Skip to content

Commit 41abbfd

Browse files
authored
qwen4exp: enable rms_norm + mul fusion (ggml-org#28896)
* qwen4exp: enable rms_norm + mul fusion * use TENSOR_ALLOW_RESHAPE
1 parent b4fa47d commit 41abbfd

1 file changed

Lines changed: 11 additions & 14 deletions

File tree

‎src/models/qwen4exp.cpp‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,8 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
157157
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
158158

159159
// there is no output_norm: the final hyper-connection mixer carries it
160-
hc_head_norm = create_tensor(tn(LLM_TENSOR_HC_HEAD_NORM, "weight"), { hc_dim }, 0);
160+
// the gammas load as [n_embd, hc] so the grouped norm multiplies them without a graph reshape
161+
hc_head_norm = create_tensor(tn(LLM_TENSOR_HC_HEAD_NORM, "weight"), { n_embd, hc }, TENSOR_ALLOW_RESHAPE);
161162
hc_head_down = create_tensor(tn(LLM_TENSOR_HC_HEAD_DOWN, "weight"), { hc_dim, hc_lr }, 0);
162163
hc_head_up = create_tensor(tn(LLM_TENSOR_HC_HEAD_UP, "weight"), { hc_lr, hc_dim }, 0);
163164

@@ -203,11 +204,11 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
203204
const int64_t conv_dim = key_dim * 2 + value_dim;
204205

205206
// two HC modules per layer: before the token mixer, before the MoE
206-
layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, 0);
207+
layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE);
207208
layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, 0);
208209
layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, 0);
209210
layer.hc_attn_inject = create_tensor(tn(LLM_TENSOR_HC_ATTN_INJECT, "weight", il), { hc_dim, hc }, 0);
210-
layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { hc_dim }, 0);
211+
layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE);
211212
layer.hc_ffn_down = create_tensor(tn(LLM_TENSOR_HC_FFN_DOWN, "weight", il), { hc_dim, hc_lr }, 0);
212213
layer.hc_ffn_up = create_tensor(tn(LLM_TENSOR_HC_FFN_UP, "weight", il), { hc_lr, hc_dim }, 0);
213214
layer.hc_ffn_inject = create_tensor(tn(LLM_TENSOR_HC_FFN_INJECT, "weight", il), { hc_dim, hc }, 0);
@@ -240,9 +241,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
240241
if (hparams.is_ple(il)) {
241242
layer.ple_key = create_tensor(tn(LLM_TENSOR_PLE_KEY, "weight", il), { n_embd, hc_dim }, 0);
242243
layer.ple_value = create_tensor(tn(LLM_TENSOR_PLE_VALUE, "weight", il), { n_embd, n_embd }, 0);
243-
layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { hc_dim }, 0);
244-
layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { hc_dim }, 0);
245-
layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { hc_dim }, 0);
244+
layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE);
245+
layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE);
246+
layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE);
246247
layer.ple_conv1d = create_tensor(tn(LLM_TENSOR_PLE_CONV1D, "weight", il), { hparams.ple_conv_kernel, hc_dim }, 0);
247248
}
248249

@@ -275,11 +276,10 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
275276
const int64_t hc_dim = hc * n_embd;
276277
const int64_t nt = x->ne[2];
277278

278-
// grouped RMSNorm: reduce over one stream, then scale all streams with the [hc_dim] gamma
279+
// grouped RMSNorm: reduce over one stream, then scale all streams with the [n_embd, hc] gamma
279280
// the converter folded each gamma to (1 + w)
280-
ggml_tensor * xn = ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps);
281+
ggml_tensor * xn = ggml_mul(ctx0, ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps), w_norm);
281282
xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt);
282-
xn = ggml_mul(ctx0, xn, w_norm);
283283
cb(xn, "hc_norm", il);
284284

285285
ggml_tensor * lo = build_lora_mm(w_down, xn);
@@ -1200,13 +1200,10 @@ ggml_tensor * llama_model_qwen4exp::graph::build_ple(
12001200
ggml_tensor * key = build_lora_mm(model.layers[il].ple_key, emb);
12011201
ggml_tensor * value = build_lora_mm(model.layers[il].ple_value, emb);
12021202

1203-
// both norms group over one hc stream, with a weight over the whole hc*n_embd layout
1203+
// both norms group over one hc stream, with a [n_embd, hc] weight
12041204
auto grouped_norm = [&](ggml_tensor * x, ggml_tensor * w) {
12051205
ggml_tensor * t = ggml_reshape_3d(ctx0, x, n_embd, hc, n_tokens);
1206-
t = ggml_rms_norm(ctx0, t, hparams.f_norm_rms_eps);
1207-
t = ggml_reshape_2d(ctx0, t, hc_dim, n_tokens);
1208-
t = ggml_mul(ctx0, t, w);
1209-
return ggml_reshape_3d(ctx0, t, n_embd, hc, n_tokens);
1206+
return ggml_mul(ctx0, ggml_rms_norm(ctx0, t, hparams.f_norm_rms_eps), w);
12101207
};
12111208

12121209
key = grouped_norm(key, model.layers[il].ple_norm_key);

0 commit comments

Comments
 (0)