qwen36-35b-a3b-nvfp4-solo.yaml

# Recipe: Qwen3.6-35B-A3B-NVFP4, solo on a single DGX Spark (GB10).
#
# Measured 2026-08-03 with llama-benchy 0.4.0 (n=3, pp=30000, tg=2000, concurrency 1):
#   generation 102.33 tok/s (std 6.50) | prefill 5,430 tok/s (std 71) | TTFT 5.58 s
#
# MEMORY, MEASURED ON THE TARGET, NOT COPIED FROM ANOTHER RECIPE:
#   Resident weights + non-torch : 46.14 GiB  (22 GB on disk; do not size from `du`)
#   KV cost                      : 14,188 bytes/token, derived as 54.75 GiB / 4,143,236
#                                  tokens from vLLM's own startup log. A layer-count
#                                  estimate predicts ~40 KiB and is wrong by 2.9x,
#                                  because this checkpoint is hybrid attention and only
#                                  a subset of layers carry a KV cache.
#   True working set             : max_num_seqs 4 x max_model_len 131072
#                                  = 524,288 tokens = 6.93 GiB
#
# WHY kv_cache_memory IS PINNED:
#   gpu_memory_utilization DID NOT bound the allocation on this vLLM build. 0.50 granted
#   60.84 GiB and vLLM allocated ~101 GiB anyway, because it sizes KV from FREE memory
#   rather than from the grant. --kv-cache-memory is the control that actually binds.
#   16 GiB is 2.3x the working set; the surplus serves the prefix cache. After pinning:
#   1,210,288 KV tokens, 9.23x concurrency at full context, 68 GB free on the box.
#
# GB10 MEMORY IS UNIFIED. A GPU fraction comes from the same 121 GB pool the OS and
# every container uses. An earlier attempt at 0.85 reserved ~103 GB, left ~18 GB for the
# OS and nine containers, and HARD-CRASHED the machine during weight load. Re-measure
# free memory before raising either value.

recipe_version: "1"
name: qwen36-35b-a3b-nvfp4-solo
description: vLLM serving Qwen3.6-35B-A3B-NVFP4 on a single DGX Spark (Marlin MoE, MTP)

model: nvidia/Qwen3.6-35B-A3B-NVFP4
container: vllm-node
cluster_only: false
solo_only: true

defaults:
  port: 8000
  host: 0.0.0.0
  tensor_parallel: 1
  gpu_memory_utilization: 0.50
  kv_cache_memory: 17179869184   # 16 GiB, measured; see header
  max_model_len: 131072
  max_num_seqs: 4
  max_num_batched_tokens: 8192

command: |
  vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 \
    --host {host} \
    --port {port} \
    --tensor-parallel-size {tensor_parallel} \
    --trust-remote-code \
    --kv-cache-dtype fp8 \
    --kv-cache-memory {kv_cache_memory} \
    --attention-backend flashinfer \
    --moe-backend marlin \
    --gpu-memory-utilization {gpu_memory_utilization} \
    --max-model-len {max_model_len} \
    --max-num-seqs {max_num_seqs} \
    --max-num-batched-tokens {max_num_batched_tokens} \
    --enable-chunked-prefill \
    --async-scheduling \
    --enable-prefix-caching \
    --speculative-config '{"method":"mtp","num_speculative_tokens":3,"moe_backend":"triton"}' \
    --load-format fastsafetensors \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen3_xml \
    --enable-auto-tool-choice