qual(cmake): support Vulcan build

Author: grencez

CMakeLists.txt

@@ -9,7 +9,6 @@ set(CMAKE_CXX_STANDARD 17)
 
 
 option(BUILD_SHARED_LIBS "Build using shared libraries" OFF)
-option(LLAMA_OPENBLAS_ON "llama: use OpenBLAS" OFF)
 option(RENDEZLLAMA_COVERAGE_ON "Enable coverage instrumentation" OFF)
 
 

Makefile

@@ -1,7 +1,4 @@
 
-# No OpenBLAS by default. Override like: make LLAMA_OPENBLAS_ON=1
-LLAMA_OPENBLAS_ON = 0
-
 BUILD_DIR = bld
 SOURCE_DIR = .
 
@@ -10,7 +7,15 @@ GODO = $(CMAKE) -E chdir
 
 CMAKE_BUILD_TYPE = RelOnHost
 CMAKE_BUILD_OPTIONS = -DCMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE)
-CMAKE_BUILD_OPTIONS += -DLLAMA_OPENBLAS_ON:BOOL=$(LLAMA_OPENBLAS_ON)
+ifdef GGML_BLAS
+CMAKE_BUILD_OPTIONS += -DGGML_BLAS:BOOL=$(GGML_BLAS)
+endif
+ifdef GGML_OPENMP
+CMAKE_BUILD_OPTIONS += -DGGML_OPENMP:BOOL=$(GGML_OPENMP)
+endif
+ifdef GGML_VULKAN
+CMAKE_BUILD_OPTIONS += -DGGML_VULKAN:BOOL=$(GGML_VULKAN)
+endif
 
 
 .PHONY: default all cmake proj \

dep/cmake_fetchcontent/llama_cpp.cmake

@@ -4,16 +4,8 @@ FetchContent_Declare(
   GIT_TAG "d9c6ce46f747189cd6238ca7699253613f77c016"
 )
 
-set(GGML_OPENMP FALSE CACHE BOOL "OpenMP off for compatibility.")
 FetchContent_MakeAvailable(LlamaCpp)
 
 set(LlamaCpp_SOURCE_DIR "${llamacpp_SOURCE_DIR}" PARENT_SCOPE)
 set(LlamaCpp_INCLUDE_DIRS "${llamacpp_SOURCE_DIR}/include" PARENT_SCOPE)
 set(LlamaCpp_LIBRARIES "$<TARGET_NAME:llama>" PARENT_SCOPE)
-
-if(LLAMA_OPENBLAS_ON)
-  find_package(OpenBLAS REQUIRED)
-  target_compile_definitions(ggml PRIVATE "GGML_USE_OPENBLAS")
-  target_include_directories(ggml PRIVATE ${OpenBLAS_INCLUDE_DIRS})
-  target_link_libraries(ggml PUBLIC ${OpenBLAS_LIBRARIES})
-endif()

src/language/inference.cc

@@ -3,7 +3,9 @@
 #include <algorithm>
 #include <cassert>
 #include <cstring>
+#include <stdexcept>
 #include <thread>
+#include <vector>
 
 #include <fildesh/fildesh.h>
 #include <fildesh/ostream.hh>
@@ -122,6 +124,20 @@ rendezllama::make_llama_context(rendezllama::ChatOptions& opt)
   if (opt.context_token_limit == 0) {
     opt.context_token_limit = opt.model_token_limit;
   }
+  float rope_freq_scale = llama_model_rope_freq_scale_train(model);
+  if (rope_freq_scale <= 0.0) {
+    rope_freq_scale = 1.0f;
+  }
+  while (
+      (unsigned)(opt.model_token_limit / rope_freq_scale)
+      <
+      opt.context_token_limit)
+  {
+    rope_freq_scale /= 2;
+  }
+  llama_model_free(model);
+  model = nullptr;
+
 
   model_params = llama_model_default_params();
   model_params.use_mlock = opt.mlock_on;
@@ -130,14 +146,33 @@ rendezllama::make_llama_context(rendezllama::ChatOptions& opt)
   llama_context_params ctx_params = llama_context_default_params();
   ctx_params.n_ctx = opt.context_token_limit;
   ctx_params.n_batch = opt.batch_count;
-  ctx_params.rope_freq_scale = llama_model_rope_freq_scale_train(model);
-  assert(ctx_params.rope_freq_scale > 0.0);
-  while (
-      (unsigned)(opt.model_token_limit / ctx_params.rope_freq_scale)
-      <
-      opt.context_token_limit)
-  {
-    ctx_params.rope_freq_scale /= 2;
+  ctx_params.rope_freq_scale = rope_freq_scale;
+
+  std::vector<float> tensor_split(llama_max_devices());
+  std::vector<llama_model_tensor_buft_override> tensor_buft_overrides(llama_max_tensor_buft_overrides());
+  std::vector<size_t> margins(llama_max_devices(), 0);
+
+  // Auto-tune parameters if possible (and not manually overridden by user yet).
+  // This helps avoid OOM crashes on Vulkan/GPU by fitting layers to available memory.
+  auto status = llama_params_fit(
+      opt.model_filename.c_str(),
+      &model_params,
+      &ctx_params,
+      tensor_split.data(),
+      tensor_buft_overrides.data(),
+      margins.data(),
+      /*n_ctx_min=*/0,
+      GGML_LOG_LEVEL_ERROR);
+
+  if (status != 0) {
+    fildesh_log_warning("llama_params_fit failed");
+  }
+
+  model = llama_model_load_from_file(
+      opt.model_filename.c_str(), model_params);
+  if (!model) {
+    fildesh_log_error("Failed to open model.");
+    return std::make_tuple(nullptr, nullptr);
   }
 
   struct llama_context* ctx = llama_init_from_model(model, ctx_params);
@@ -367,14 +402,6 @@ Inference::commit_to_context(
         opt.batch_count,
         chat_traj.token_count() - chat_traj.context_token_count_);
 
-#if LLAMA_OPENBLAS_ON
-    if (n < 32) {
-      llama_set_n_threads(ctx, thread_count_, batch_thread_count_);
-    }
-    else {
-      llama_set_n_threads(ctx, thread_count_, 1);
-    }
-#endif
     chat_disp.show_new(chat_traj.context_token_count_ + n, chat_traj, vocabulary_);
 
     if (!batch_.token || (unsigned)batch_.n_tokens < n) {
@@ -445,4 +472,3 @@ Inference::sample_to_trajectory(
   llama_sampler_accept(smpl_, chat_traj.token());
   token_count_ += 1;
 }
-