Skip to content

Commit 987e3df

Browse files
authored
fix(example): retain recurrent state for MTP rollback (#2358)
1 parent 8a0810f commit 987e3df

2 files changed

Lines changed: 16 additions & 1 deletion

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
- fix(example): retain recurrent state for server MTP rollback
1011
- feat: update llama.cpp to ggml-org/llama.cpp@adb55e514
1112

1213
## [0.3.34]

‎examples/server/server.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11362,6 +11362,11 @@ def __init__(
1136211362
"speculative decoding is only supported for attention models"
1136311363
)
1136411364
n_ctx_train = int(llama_cpp.llama_model_n_ctx_train(llama_model))
11365+
target_n_rs_seq = (
11366+
max(1, draft_model_num_pred_tokens)
11367+
if normalized_draft_model == "draft-mtp"
11368+
else None
11369+
)
1136511370

1136611371
context_params = self.build_context_params(
1136711372
n_ctx=n_ctx if n_ctx is not None else n_ctx_train,
@@ -11389,7 +11394,7 @@ def __init__(
1138911394
type_k=type_k,
1139011395
type_v=type_v,
1139111396
kv_unified=kv_unified,
11392-
n_rs_seq=None,
11397+
n_rs_seq=target_n_rs_seq,
1139311398
ctx_type=None,
1139411399
)
1139511400
ctx = llama_cpp.llama_init_from_model(llama_model, context_params)
@@ -11419,6 +11424,15 @@ def __init__(
1141911424
"MTP requires runtime n_batch to fit the pending token plus draft tokens "
1142011425
f"(required {required_mtp_batch}, got {self.n_batch})"
1142111426
)
11427+
if (
11428+
target_n_rs_seq is not None
11429+
and self.exact_checkpoints_only
11430+
and self.n_rs_seq < target_n_rs_seq
11431+
):
11432+
raise RuntimeError(
11433+
"MTP requires retained recurrent-state slots for rollback "
11434+
f"(required {target_n_rs_seq}, got {self.n_rs_seq})"
11435+
)
1142211436
self.n_ctx_train = n_ctx_train
1142311437
self.n_vocab = int(llama_cpp.llama_vocab_n_tokens(self.vocab))
1142411438
self.n_embd = int(llama_cpp.llama_model_n_embd(self.llama_model))

0 commit comments

Comments
 (0)