forge-alloy/examples/qwen3.5-4b-code-balanced.alloy.json at main · CambrianTech/forge-alloy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
{
  "name": "qwen3.5-4b-code-balanced",
  "version": "1.0.0",
  "description": "Balanced code forge for Qwen3.5-4B. 30% entropy pruning, 3 cycles, GGUF output for MacBook and iPhone.",
  "author": "continuum-ai",
  "tags": ["code", "qwen3.5", "4b", "balanced", "macbook", "iphone", "forge-alloy"],
  "license": "apache-2.0",

  "source": {
    "baseModel": "Qwen/Qwen3.5-4B",
    "architecture": "qwen3_5",
    "isMoE": false
  },

  "stages": [
    {
      "type": "prune",
      "strategy": "entropy",
      "level": 0.3,
      "minHeadsPerLayer": 4,
      "minKvHeadsPerLayer": 2,
      "analysisSteps": 200
    },
    {
      "type": "train",
      "domain": "code",
      "dataset": "m-a-p/CodeFeedback-Filtered-Instruction",
      "steps": 1000,
      "learningRate": "2e-4",
      "batchSize": 4,
      "gradientAccumulation": 4,
      "scheduler": "cosine",
      "warmupRatio": 0.03,
      "weightDecay": 0.01,
      "maxGradientNorm": 1.0,
      "precision": "bf16",
      "sequenceLength": 2048,
      "optimizations": ["flash_attention", "gradient_checkpointing"]
    },
    {
      "type": "compact",
      "deadThreshold": 0.1,
      "dormantThreshold": 0.2,
      "lowThreshold": 0.3,
      "mediumThreshold": 0.5,
      "highThreshold": 0.7,
      "enableQuantization": true
    },
    {
      "type": "quant",
      "format": "gguf",
      "quantTypes": ["Q4_K_M", "Q8_0"],
      "deviceTargets": ["macbookair", "iphone", "rtx3090"]
    },
    {
      "type": "eval",
      "benchmarks": [
        { "name": "humaneval", "submitToLeaderboard": true },
        { "name": "mmlu", "subset": "mmlu-pro", "nShot": 5, "submitToLeaderboard": false },
        { "name": "gsm8k", "nShot": 8, "submitToLeaderboard": false }
      ],
      "passingThreshold": 60,
      "compareToBase": true
    },
    {
      "type": "publish",
      "org": "continuum-ai",
      "repoNameTemplate": "{base}-{domain}-forged",
      "includeAlloy": true,
      "cardFromBenchmarks": true,
      "tags": ["continuum", "forged", "experiential-plasticity", "head-pruning"],
      "private": false
    }
  ],

  "cycles": 3,

  "hardware": {
    "minVramGb": 8,
    "recommendedVramGb": 24,
    "estimatedDurationMinutes": 45,
    "supportsCPU": false,
    "testedOn": ["RTX 5090", "RTX 4090"]
  },

  "outputs": {
    "produces": [
      { "type": "safetensors", "description": "Forged base weights (full precision)" },
      { "type": "gguf", "description": "Q4_K_M for MacBook Air, Q8_0 for desktop" },
      { "type": "model-card", "description": "Auto-generated from benchmark results" },
      { "type": "alloy", "description": "This alloy (for reproducibility)" }
    ]
  }
}