ninfer-3080/tools/windows/profiles.json at master · ahnafnafee/ninfer-3080 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
{
  "port": 8080,
  "profiles": {
    "reason64": {
      "note": "Default: Bonsai 2 Heretic, 64K minimum, low reasoning capped at 2048, temperature 0.2, RK4 KV, FP32 recurrent state; draft-head VRAM reserved for cache precision",
      "minimumContext": 65536,
      "exe": "{repo}\\build-ninja\\apps\\ninfer-serve.exe",
      "args": ["{models}\\Ternary-Bonsai-2-27B-Heretic-ninfer.ninfer", "--model-id", "bonsai2-heretic",
               "--max-context", "65536", "--kv-capacity", "65536", "--kv-dtype", "rk4v4",
               "--prefill-chunk", "128", "--default-reasoning-effort", "low", "--temperature", "0.2",
               "--default-thinking-budget", "2048",
               "--device-state-slots", "0", "--host-state-slots", "4",
               "--default-max-tokens", "8192",
               "--max-concurrency", "1"]
    },
    "reason8": {
      "note": "Short-context precision option: medium reasoning, INT8 KV at 8K, FP32 recurrent state, MTP 3 drafts",
      "exe": "{repo}\\build-ninja\\apps\\ninfer-serve.exe",
      "args": ["{models}\\Ternary-Bonsai-2-27B-Heretic-ninfer.ninfer", "--model-id", "bonsai2-heretic",
               "--max-context", "8192", "--kv-capacity", "8192", "--kv-dtype", "int8",
               "--default-reasoning-effort", "medium",
               "--spec", "mtp", "--draft-tokens", "3", "--max-concurrency", "1"]
    },
    "mtp": {
      "note": "Ternary Bonsai 2 Heretic, MTP 3 drafts, 64K, rk2v4-e8 KV (RTX 3080: 139 tok/s short, 94 full)",
      "exe": "{repo}\\build-ninja\\apps\\ninfer-serve.exe",
      "args": ["{models}\\Ternary-Bonsai-2-27B-Heretic-ninfer.ninfer", "--model-id", "bonsai2-heretic",
               "--max-context", "65536", "--kv-capacity", "65536", "--kv-dtype", "rk2v4-e8", "--gdn-state-fp16",
               "--spec", "mtp", "--draft-tokens", "3", "--max-concurrency", "1"]
    },
    "96k": {
      "note": "Ternary Bonsai 2 Heretic, no speculation, 96K, rk2v4-e8 KV (RTX 3080: 70 tok/s short, 53 full)",
      "exe": "{repo}\\build-ninja\\apps\\ninfer-serve.exe",
      "args": ["{models}\\Ternary-Bonsai-2-27B-Heretic-ninfer.ninfer", "--model-id", "bonsai2-heretic-96k",
               "--max-context", "98304", "--kv-capacity", "98304", "--kv-dtype", "rk2v4-e8", "--gdn-state-fp16",
               "--max-concurrency", "1"]
    },
    "precision16": {
      "note": "Ternary Bonsai 2 Heretic, MTP 3 drafts, 16K, INT8 KV for higher cache precision on a 10 GB RTX 3080",
      "exe": "{repo}\\build-ninja\\apps\\ninfer-serve.exe",
      "args": ["{models}\\Ternary-Bonsai-2-27B-Heretic-ninfer.ninfer", "--model-id", "bonsai2-heretic-precision",
               "--max-context", "16384", "--kv-capacity", "16384", "--kv-dtype", "int8", "--gdn-state-fp16",
               "--spec", "mtp", "--draft-tokens", "3", "--max-concurrency", "1"]
    },
    "mtp48": {
      "note": "Ternary Bonsai 2 Heretic, MTP 3 drafts, 48K, rk4v4 KV for higher KV precision (RTX 3080: 137 tok/s short)",
      "exe": "{repo}\\build-ninja\\apps\\ninfer-serve.exe",
      "args": ["{models}\\Ternary-Bonsai-2-27B-Heretic-ninfer.ninfer", "--model-id", "bonsai2-heretic-48k",
               "--max-context", "49152", "--kv-capacity", "49152", "--kv-dtype", "rk4v4", "--gdn-state-fp16",
               "--spec", "mtp", "--draft-tokens", "3", "--max-concurrency", "1"]
    }
  }
}