feat: update llama.cpp to 11924d4c1

Author: abetlen

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-- feat: update llama.cpp to ggml-org/llama.cpp@d2a818231
+- feat: update llama.cpp to ggml-org/llama.cpp@11924d4c1
 
 ## [0.3.34]
 

examples/low_level_api/low_level_api_chat_cpp.py

@@ -76,7 +76,9 @@ def __init__(self, params: GptParams) -> None:
         self.lparams.n_parts = self.params.n_parts
         self.lparams.seed = self.params.seed
         self.lparams.memory_f16 = self.params.memory_f16
-        if self.params.use_mlock:
+        if self.params.use_mmap and self.params.use_mlock:
+            self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK
+        elif self.params.use_mlock:
             self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
         elif self.params.use_mmap:
             self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP

examples/server/server.py

@@ -11312,6 +11312,7 @@ def __init__(
                 vocab_only=vocab_only,
                 use_mmap=use_mmap,
                 use_mlock=use_mlock,
+                load_mtp=draft_model == "draft-mtp",
                 kv_overrides=kv_overrides,
             )
         )
@@ -11590,6 +11591,7 @@ def build_model_params(
         vocab_only: Optional[bool],
         use_mmap: Optional[bool],
         use_mlock: Optional[bool],
+        load_mtp: bool,
         kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]],
     ) -> Tuple[Any, Optional[Any], Optional[Any]]:
         model_params = llama_cpp.llama_model_default_params()
@@ -11611,7 +11613,10 @@ def build_model_params(
             model_params.tensor_split = tensor_split_ref
         if vocab_only is not None:
             model_params.vocab_only = vocab_only
-        if use_mlock:
+        model_params.load_mtp = load_mtp
+        if use_mlock and use_mmap is not False:
+            model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK
+        elif use_mlock:
             model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
         elif use_mmap is not None:
             model_params.load_mode = (

llama_cpp/llama.py

@@ -244,8 +244,9 @@ def __init__(
             )  # keep a reference to the array so it is not gc'd
             self.model_params.tensor_split = self._c_tensor_split
         self.model_params.vocab_only = vocab_only
-        if lora_path is not None:
-            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE
+        use_mmap = use_mmap and lora_path is None
+        if use_mmap and use_mlock:
+            self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK
         elif use_mlock:
             self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
         elif use_mmap:
@@ -2149,8 +2150,15 @@ def __getstate__(self):
             tensor_split=self.tensor_split,
             vocab_only=self.model_params.vocab_only,
             use_mmap=self.model_params.load_mode
-            in (llama_cpp.LLAMA_LOAD_MODE_MMAP, llama_cpp.LLAMA_LOAD_MODE_MLOCK),
-            use_mlock=self.model_params.load_mode == llama_cpp.LLAMA_LOAD_MODE_MLOCK,
+            in (
+                llama_cpp.LLAMA_LOAD_MODE_MMAP,
+                llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK,
+            ),
+            use_mlock=self.model_params.load_mode
+            in (
+                llama_cpp.LLAMA_LOAD_MODE_MLOCK,
+                llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK,
+            ),
             kv_overrides=self.kv_overrides,
             # Context Params
             seed=self._seed,

llama_cpp/llama_cpp.py

@@ -521,15 +521,17 @@ def _warn_deprecated(symbol: str, hint: str) -> None:
 
 
 # enum llama_load_mode {
-#     LLAMA_LOAD_MODE_NONE      = 0, // no special loading mode
-#     LLAMA_LOAD_MODE_MMAP      = 1, // memory map the model
-#     LLAMA_LOAD_MODE_MLOCK     = 2, // mmap + force system to keep model in RAM rather than swapping or compressing
-#     LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available
+#     LLAMA_LOAD_MODE_NONE       = 0, // no special loading mode
+#     LLAMA_LOAD_MODE_MMAP       = 1, // memory map the model
+#     LLAMA_LOAD_MODE_MLOCK      = 2, // force system to keep model in RAM rather than swapping or compressing
+#     LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing
+#     LLAMA_LOAD_MODE_DIRECT_IO  = 4, // use direct I/O if available
 # };
 LLAMA_LOAD_MODE_NONE = 0
 LLAMA_LOAD_MODE_MMAP = 1
 LLAMA_LOAD_MODE_MLOCK = 2
-LLAMA_LOAD_MODE_DIRECT_IO = 3
+LLAMA_LOAD_MODE_MMAP_MLOCK = 3
+LLAMA_LOAD_MODE_DIRECT_IO = 4
 
 
 # enum llama_context_type {
@@ -829,6 +831,7 @@ class llama_model_imatrix_data(ctypes.Structure):
 #     bool use_extra_bufts; // use extra buffer types (used for weight repacking)
 #     bool no_host;         // bypass host buffer allowing extra buffers to be used
 #     bool no_alloc;        // only load metadata and simulate memory allocations
+#     bool load_mtp;        // whether to load MTP layers
 # };
 class llama_model_params(ctypes.Structure):
     """Parameters for llama_model
@@ -848,7 +851,8 @@ class llama_model_params(ctypes.Structure):
         check_tensors (bool): validate model tensor data
         use_extra_bufts (bool): use extra buffer types (used for weight repacking)
         no_host (bool): bypass host buffer allowing extra buffers to be used
-        no_alloc (bool): only load metadata and simulate memory allocations"""
+        no_alloc (bool): only load metadata and simulate memory allocations
+        load_mtp (bool): whether to load MTP layers"""
 
     if TYPE_CHECKING:
         devices: CtypesArray[ctypes.c_void_p]  # NOTE: unused
@@ -868,6 +872,7 @@ class llama_model_params(ctypes.Structure):
         use_extra_bufts: bool
         no_host: bool
         no_alloc: bool
+        load_mtp: bool
 
     _fields_ = [
         ("devices", ctypes.c_void_p),  # NOTE: unnused
@@ -885,6 +890,7 @@ class llama_model_params(ctypes.Structure):
         ("use_extra_bufts", ctypes.c_bool),
         ("no_host", ctypes.c_bool),
         ("no_alloc", ctypes.c_bool),
+        ("load_mtp", ctypes.c_bool),
     ]
 
 
@@ -3549,6 +3555,20 @@ def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ...
 def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ...
 
 
+# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens)
+# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens);
+@ctypes_function(
+    "llama_vocab_get_suppress_tokens",
+    [llama_vocab_p_ctypes, ctypes.POINTER(ctypes.c_int32)],
+    ctypes.POINTER(llama_token),
+)
+def llama_vocab_get_suppress_tokens(
+    vocab: llama_vocab_p,
+    n_suppress_tokens: CtypesPointer[ctypes.c_int32],
+    /,
+) -> Optional[CtypesPointer[llama_token]]: ...
+
+
 # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
 @ctypes_function(
     "llama_vocab_fim_pre",

tests/test_llama.py

@@ -103,7 +103,9 @@ def test_real_model(llama_cpp_model_path):
     assert os.path.exists(llama_cpp_model_path)
 
     params = llama_cpp.llama_model_default_params()
-    if llama_cpp.llama_supports_mlock():
+    if llama_cpp.llama_supports_mmap() and llama_cpp.llama_supports_mlock():
+        params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK
+    elif llama_cpp.llama_supports_mlock():
         params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK
     elif llama_cpp.llama_supports_mmap():
         params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP