Skip to content

Commit 4fbdabd

Browse files
authored
model: using single llm_build per arch (ggml-org#21970)
* model: using single llm_build per arch * fix merge * nits
1 parent e45dbde commit 4fbdabd

7 files changed

Lines changed: 139 additions & 128 deletions

File tree

‎src/CMakeLists.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ add_library(llama
9292
models/lfm2.cpp
9393
models/llada-moe.cpp
9494
models/llada.cpp
95-
models/llama-iswa.cpp
9695
models/llama.cpp
96+
models/llama4.cpp
9797
models/maincoder.cpp
9898
models/mamba-base.cpp
9999
models/mamba.cpp
@@ -145,8 +145,8 @@ add_library(llama
145145
models/starcoder.cpp
146146
models/starcoder2.cpp
147147
models/step35-iswa.cpp
148-
models/t5-dec.cpp
149-
models/t5-enc.cpp
148+
models/t5.cpp
149+
models/t5encoder.cpp
150150
models/wavtokenizer-dec.cpp
151151
models/xverse.cpp
152152
)

‎src/llama-model.cpp‎

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1271,8 +1271,7 @@ void llama_model::load_hparams(llama_model_loader & ml) {
12711271
}
12721272
// Set non-causal attention for diffusion models
12731273
hparams.causal_attn = false;
1274-
}
1275-
break;
1274+
} break;
12761275
case LLM_ARCH_LLADA:
12771276
{
12781277
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
@@ -1286,8 +1285,7 @@ void llama_model::load_hparams(llama_model_loader & ml) {
12861285
}
12871286
// Set non-causal attention for diffusion models
12881287
hparams.causal_attn = false;
1289-
}
1290-
break;
1288+
} break;
12911289
case LLM_ARCH_LLADA_MOE:
12921290
{
12931291
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
@@ -8553,9 +8551,9 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {
85538551
case LLM_ARCH_LLAMA4:
85548552
{
85558553
if (hparams.swa_type == LLAMA_SWA_TYPE_NONE) {
8556-
llm = std::make_unique<llm_build_llama<false>>(*this, params);
8554+
llm = std::make_unique<llm_build_llama4<false>>(*this, params);
85578555
} else {
8558-
llm = std::make_unique<llm_build_llama_iswa>(*this, params);
8556+
llm = std::make_unique<llm_build_llama4<true>>(*this, params);
85598557
}
85608558
} break;
85618559
case LLM_ARCH_LLAMA_EMBED:
@@ -8633,23 +8631,19 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {
86338631
case LLM_ARCH_DREAM:
86348632
{
86358633
llm = std::make_unique<llm_build_dream>(*this, params);
8636-
}
8637-
break;
8634+
} break;
86388635
case LLM_ARCH_LLADA:
86398636
{
86408637
llm = std::make_unique<llm_build_llada>(*this, params);
8641-
}
8642-
break;
8638+
} break;
86438639
case LLM_ARCH_LLADA_MOE:
86448640
{
86458641
llm = std::make_unique<llm_build_llada_moe>(*this, params);
8646-
}
8647-
break;
8642+
} break;
86488643
case LLM_ARCH_RND1:
86498644
{
86508645
llm = std::make_unique<llm_build_rnd1>(*this, params);
8651-
}
8652-
break;
8646+
} break;
86538647
case LLM_ARCH_QWEN2VL:
86548648
{
86558649
llm = std::make_unique<llm_build_qwen2vl>(*this, params);
@@ -8839,21 +8833,20 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {
88398833
{
88408834
switch (params.gtype) {
88418835
case LLM_GRAPH_TYPE_ENCODER:
8842-
llm = std::make_unique<llm_build_t5_enc>(*this, params);
8836+
llm = std::make_unique<llm_build_t5<true>>(*this, params);
88438837
break;
88448838
case LLM_GRAPH_TYPE_DEFAULT:
88458839
case LLM_GRAPH_TYPE_DECODER:
8846-
llm = std::make_unique<llm_build_t5_dec>(*this, params);
8840+
llm = std::make_unique<llm_build_t5<false>>(*this, params);
88478841
break;
88488842
default:
88498843
GGML_ABORT("invalid graph type");
88508844
};
88518845
} break;
88528846
case LLM_ARCH_T5ENCODER:
88538847
{
8854-
llm = std::make_unique<llm_build_t5_enc>(*this, params);
8855-
}
8856-
break;
8848+
llm = std::make_unique<llm_build_t5encoder>(*this, params);
8849+
} break;
88578850
case LLM_ARCH_JAIS:
88588851
{
88598852
llm = std::make_unique<llm_build_jais>(*this, params);
Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "models.h"
22

3-
llm_build_llama_iswa::llm_build_llama_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
3+
template <bool iswa>
4+
llm_build_llama4<iswa>::llm_build_llama4(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
45
const int64_t n_embd_head = hparams.n_embd_head_v();
56

67
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
@@ -18,7 +19,14 @@ llm_build_llama_iswa::llm_build_llama_iswa(const llama_model & model, const llm_
1819
ggml_tensor * inp_attn_scale = nullptr;
1920
inp_attn_scale = build_inp_attn_scale();
2021

21-
auto * inp_attn = build_attn_inp_kv_iswa();
22+
using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;
23+
inp_attn_type * inp_attn = nullptr;
24+
25+
if constexpr (iswa) {
26+
inp_attn = build_attn_inp_kv_iswa();
27+
} else {
28+
inp_attn = build_attn_inp_kv();
29+
}
2230

2331
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
2432

@@ -157,3 +165,7 @@ llm_build_llama_iswa::llm_build_llama_iswa(const llama_model & model, const llm_
157165

158166
ggml_build_forward_expand(gf, cur);
159167
}
168+
169+
// Explicit template instantiations
170+
template struct llm_build_llama4<false>;
171+
template struct llm_build_llama4<true>;

‎src/models/models.h‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -407,8 +407,9 @@ struct llm_build_llama : public llm_graph_context {
407407
llm_build_llama(const llama_model & model, const llm_graph_params & params);
408408
};
409409

410-
struct llm_build_llama_iswa : public llm_graph_context {
411-
llm_build_llama_iswa(const llama_model & model, const llm_graph_params & params);
410+
template <bool iswa>
411+
struct llm_build_llama4 : public llm_graph_context {
412+
llm_build_llama4(const llama_model & model, const llm_graph_params & params);
412413
};
413414

414415
struct llm_build_maincoder : public llm_graph_context {
@@ -495,7 +496,7 @@ struct llm_build_phi2 : public llm_graph_context {
495496
llm_build_phi2(const llama_model & model, const llm_graph_params & params);
496497
};
497498

498-
template<bool iswa>
499+
template <bool iswa>
499500
struct llm_build_phi3 : public llm_graph_context {
500501
llm_build_phi3(const llama_model & model, const llm_graph_params & params);
501502
};
@@ -701,12 +702,13 @@ struct llm_build_step35_iswa : public llm_graph_context {
701702
llm_build_step35_iswa(const llama_model & model, const llm_graph_params & params);
702703
};
703704

704-
struct llm_build_t5_dec : public llm_graph_context {
705-
llm_build_t5_dec(const llama_model & model, const llm_graph_params & params);
705+
template <bool is_enc>
706+
struct llm_build_t5 : public llm_graph_context {
707+
llm_build_t5(const llama_model & model, const llm_graph_params & params);
706708
};
707709

708-
struct llm_build_t5_enc : public llm_graph_context {
709-
llm_build_t5_enc(const llama_model & model, const llm_graph_params & params);
710+
struct llm_build_t5encoder : public llm_build_t5<true> {
711+
llm_build_t5encoder(const llama_model & model, const llm_graph_params & params);
710712
};
711713

712714
struct llm_build_wavtokenizer_dec : public llm_graph_context {

‎src/models/t5-enc.cpp‎

Lines changed: 0 additions & 96 deletions
This file was deleted.
Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "models.h"
22

3-
llm_build_t5_dec::llm_build_t5_dec(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
3+
template <>
4+
llm_build_t5<false>::llm_build_t5(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
45
const int64_t n_embd_head = hparams.n_embd_head_v();
56
//const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
67

@@ -153,3 +154,99 @@ llm_build_t5_dec::llm_build_t5_dec(const llama_model & model, const llm_graph_pa
153154

154155
ggml_build_forward_expand(gf, cur);
155156
}
157+
158+
template <>
159+
llm_build_t5<true>::llm_build_t5(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
160+
const int64_t n_embd_head = hparams.n_embd_head_v();
161+
162+
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
163+
164+
ggml_tensor * cur;
165+
ggml_tensor * inpL;
166+
167+
inpL = build_inp_embd(model.tok_embd);
168+
169+
ggml_tensor * pos_bucket_enc = build_inp_pos_bucket_enc();
170+
171+
auto * inp_attn = build_attn_inp_no_cache();
172+
173+
ggml_tensor * inp_out_ids = build_inp_out_ids();
174+
175+
for (int il = 0; il < n_layer; ++il) {
176+
ggml_tensor * inpSA = inpL;
177+
178+
// norm
179+
cur = build_norm(inpL,
180+
model.layers[il].attn_norm_enc, NULL,
181+
LLM_NORM_RMS, il);
182+
cb(cur, "attn_norm", il);
183+
184+
// self-attention
185+
{
186+
ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_enc, cur);
187+
cb(Qcur, "Qcur", il);
188+
189+
ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_enc, cur);
190+
cb(Kcur, "Kcur", il);
191+
192+
ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_enc, cur);
193+
cb(Vcur, "Vcur", il);
194+
195+
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
196+
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
197+
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
198+
199+
ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;
200+
ggml_tensor * kq_b = build_pos_bias(pos_bucket_enc, attn_rel_b);
201+
202+
cur = build_attn(inp_attn,
203+
model.layers[il].wo_enc, nullptr, nullptr,
204+
Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);
205+
cb(cur, "kqv_out", il);
206+
}
207+
if (il == n_layer - 1 && inp_out_ids) {
208+
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
209+
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
210+
}
211+
ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
212+
cb(ffn_inp, "ffn_inp", il);
213+
214+
// feed-forward network
215+
{
216+
cur = build_norm(ffn_inp,
217+
model.layers[il].ffn_norm_enc, NULL,
218+
LLM_NORM_RMS, il);
219+
cb(cur, "ffn_norm", il);
220+
221+
// T5 uses relu, flan-T5 uses gelu-gated
222+
cur = build_ffn(cur,
223+
model.layers[il].ffn_up_enc, NULL, NULL,
224+
model.layers[il].ffn_gate_enc, NULL, NULL,
225+
model.layers[il].ffn_down_enc, NULL, NULL,
226+
NULL,
227+
model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
228+
model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
229+
il);
230+
cb(cur, "ffn_out", il);
231+
}
232+
cur = ggml_add(ctx0, cur, ffn_inp);
233+
cb(cur, "ffn_out", il);
234+
235+
cur = build_cvec(cur, il);
236+
cb(cur, "l_out", il);
237+
238+
// input for next layer
239+
inpL = cur;
240+
}
241+
cur = inpL;
242+
cb(cur, "result_embd", -1);
243+
244+
cur = build_norm(cur,
245+
model.output_norm_enc, NULL,
246+
LLM_NORM_RMS, -1);
247+
248+
cb(cur, "result_norm", -1);
249+
res->t_embd = cur;
250+
251+
ggml_build_forward_expand(gf, cur);
252+
}

‎src/models/t5encoder.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#include "models.h"
2+
3+
llm_build_t5encoder::llm_build_t5encoder(const llama_model & model, const llm_graph_params & params) : llm_build_t5<true>(model, params) {}

0 commit comments

Comments
 (0)