@@ -157,7 +157,8 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
157157 tok_embd = create_tensor (tn (LLM_TENSOR_TOKEN_EMBD , " weight" ), { n_embd, n_vocab }, 0 );
158158
159159 // there is no output_norm: the final hyper-connection mixer carries it
160- hc_head_norm = create_tensor (tn (LLM_TENSOR_HC_HEAD_NORM , " weight" ), { hc_dim }, 0 );
160+ // the gammas load as [n_embd, hc] so the grouped norm multiplies them without a graph reshape
161+ hc_head_norm = create_tensor (tn (LLM_TENSOR_HC_HEAD_NORM , " weight" ), { n_embd, hc }, TENSOR_ALLOW_RESHAPE );
161162 hc_head_down = create_tensor (tn (LLM_TENSOR_HC_HEAD_DOWN , " weight" ), { hc_dim, hc_lr }, 0 );
162163 hc_head_up = create_tensor (tn (LLM_TENSOR_HC_HEAD_UP , " weight" ), { hc_lr, hc_dim }, 0 );
163164
@@ -203,11 +204,11 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
203204 const int64_t conv_dim = key_dim * 2 + value_dim;
204205
205206 // two HC modules per layer: before the token mixer, before the MoE
206- layer.hc_attn_norm = create_tensor (tn (LLM_TENSOR_HC_ATTN_NORM , " weight" , il), { hc_dim }, 0 );
207+ layer.hc_attn_norm = create_tensor (tn (LLM_TENSOR_HC_ATTN_NORM , " weight" , il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE );
207208 layer.hc_attn_down = create_tensor (tn (LLM_TENSOR_HC_ATTN_DOWN , " weight" , il), { hc_dim, hc_lr }, 0 );
208209 layer.hc_attn_up = create_tensor (tn (LLM_TENSOR_HC_ATTN_UP , " weight" , il), { hc_lr, hc_dim }, 0 );
209210 layer.hc_attn_inject = create_tensor (tn (LLM_TENSOR_HC_ATTN_INJECT , " weight" , il), { hc_dim, hc }, 0 );
210- layer.hc_ffn_norm = create_tensor (tn (LLM_TENSOR_HC_FFN_NORM , " weight" , il), { hc_dim }, 0 );
211+ layer.hc_ffn_norm = create_tensor (tn (LLM_TENSOR_HC_FFN_NORM , " weight" , il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE );
211212 layer.hc_ffn_down = create_tensor (tn (LLM_TENSOR_HC_FFN_DOWN , " weight" , il), { hc_dim, hc_lr }, 0 );
212213 layer.hc_ffn_up = create_tensor (tn (LLM_TENSOR_HC_FFN_UP , " weight" , il), { hc_lr, hc_dim }, 0 );
213214 layer.hc_ffn_inject = create_tensor (tn (LLM_TENSOR_HC_FFN_INJECT , " weight" , il), { hc_dim, hc }, 0 );
@@ -240,9 +241,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
240241 if (hparams.is_ple (il)) {
241242 layer.ple_key = create_tensor (tn (LLM_TENSOR_PLE_KEY , " weight" , il), { n_embd, hc_dim }, 0 );
242243 layer.ple_value = create_tensor (tn (LLM_TENSOR_PLE_VALUE , " weight" , il), { n_embd, n_embd }, 0 );
243- layer.ple_norm_key = create_tensor (tn (LLM_TENSOR_PLE_NORM_KEY , " weight" , il), { hc_dim }, 0 );
244- layer.ple_norm_query = create_tensor (tn (LLM_TENSOR_PLE_NORM_QUERY , " weight" , il), { hc_dim }, 0 );
245- layer.ple_norm_conv = create_tensor (tn (LLM_TENSOR_PLE_NORM_CONV , " weight" , il), { hc_dim }, 0 );
244+ layer.ple_norm_key = create_tensor (tn (LLM_TENSOR_PLE_NORM_KEY , " weight" , il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE );
245+ layer.ple_norm_query = create_tensor (tn (LLM_TENSOR_PLE_NORM_QUERY , " weight" , il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE );
246+ layer.ple_norm_conv = create_tensor (tn (LLM_TENSOR_PLE_NORM_CONV , " weight" , il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE );
246247 layer.ple_conv1d = create_tensor (tn (LLM_TENSOR_PLE_CONV1D , " weight" , il), { hparams.ple_conv_kernel , hc_dim }, 0 );
247248 }
248249
@@ -275,11 +276,10 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
275276 const int64_t hc_dim = hc * n_embd;
276277 const int64_t nt = x->ne [2 ];
277278
278- // grouped RMSNorm: reduce over one stream, then scale all streams with the [hc_dim ] gamma
279+ // grouped RMSNorm: reduce over one stream, then scale all streams with the [n_embd, hc ] gamma
279280 // the converter folded each gamma to (1 + w)
280- ggml_tensor * xn = ggml_rms_norm (ctx0, x, hparams.f_norm_rms_eps );
281+ ggml_tensor * xn = ggml_mul (ctx0, ggml_rms_norm (ctx0, x, hparams.f_norm_rms_eps ), w_norm );
281282 xn = ggml_reshape_2d (ctx0, xn, hc_dim, nt);
282- xn = ggml_mul (ctx0, xn, w_norm);
283283 cb (xn, " hc_norm" , il);
284284
285285 ggml_tensor * lo = build_lora_mm (w_down, xn);
@@ -1200,13 +1200,10 @@ ggml_tensor * llama_model_qwen4exp::graph::build_ple(
12001200 ggml_tensor * key = build_lora_mm (model.layers [il].ple_key , emb);
12011201 ggml_tensor * value = build_lora_mm (model.layers [il].ple_value , emb);
12021202
1203- // both norms group over one hc stream, with a weight over the whole hc*n_embd layout
1203+ // both norms group over one hc stream, with a [n_embd, hc] weight
12041204 auto grouped_norm = [&](ggml_tensor * x, ggml_tensor * w) {
12051205 ggml_tensor * t = ggml_reshape_3d (ctx0, x, n_embd, hc, n_tokens);
1206- t = ggml_rms_norm (ctx0, t, hparams.f_norm_rms_eps );
1207- t = ggml_reshape_2d (ctx0, t, hc_dim, n_tokens);
1208- t = ggml_mul (ctx0, t, w);
1209- return ggml_reshape_3d (ctx0, t, n_embd, hc, n_tokens);
1206+ return ggml_mul (ctx0, ggml_rms_norm (ctx0, t, hparams.f_norm_rms_eps ), w);
12101207 };
12111208
12121209 key = grouped_norm (key, model.layers [il].ple_norm_key );
0 commit comments