|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#include <unistd.h> |
| 10 | +#include <csignal> |
| 11 | +#include <cstdint> |
| 12 | +#include <iostream> |
| 13 | +#include <limits> |
| 14 | +#include <memory> |
| 15 | +#include <stdexcept> |
| 16 | + |
| 17 | +#include <gflags/gflags.h> |
| 18 | + |
| 19 | +#include <executorch/examples/llm_server/cpp/multiplexed_worker.h> |
| 20 | +#include <executorch/extension/llm/batching/decode_first_scheduler.h> |
| 21 | +#include <executorch/extension/llm/batching/module_executor.h> |
| 22 | +#include <executorch/extension/llm/cache/cache_registry.h> |
| 23 | +#include <executorch/extension/llm/runner/llm_runner_helper.h> |
| 24 | +#include <executorch/extension/llm/runner/model_metadata.h> |
| 25 | +#include <executorch/extension/module/module.h> |
| 26 | + |
| 27 | +DEFINE_string(pte, "", "Packed off-graph-cache text model program"); |
| 28 | +DEFINE_string(tokenizer, "", "Tokenizer matching the program"); |
| 29 | +DEFINE_int32(max_sessions, 4, "Maximum logical serving sessions"); |
| 30 | +DEFINE_int32( |
| 31 | + prefix_cache_entries, |
| 32 | + 0, |
| 33 | + "Greedy creation-only prompt snapshots; zero disables prefix caching"); |
| 34 | +DEFINE_int32(max_session_tokens, 2048, "Maximum retained tokens per session"); |
| 35 | +DEFINE_int32(max_decode_sequences, 2, "Decode slots in one packed forward"); |
| 36 | +DEFINE_int32( |
| 37 | + max_inflight_requests, |
| 38 | + 64, |
| 39 | + "Bounded wire generation/lifecycle capacity"); |
| 40 | + |
| 41 | +int main(int argc, char** argv) { |
| 42 | + gflags::ParseCommandLineFlags(&argc, &argv, true); |
| 43 | + if (FLAGS_pte.empty() || FLAGS_tokenizer.empty() || FLAGS_max_sessions <= 0 || |
| 44 | + FLAGS_max_session_tokens <= 0 || FLAGS_max_decode_sequences <= 0 || |
| 45 | + FLAGS_max_inflight_requests <= 0 || FLAGS_prefix_cache_entries < 0) { |
| 46 | + std::cerr |
| 47 | + << "--pte and --tokenizer are required; capacity limits must be positive " |
| 48 | + "and prefix_cache_entries nonnegative\n"; |
| 49 | + return 1; |
| 50 | + } |
| 51 | + const auto physical_sessions = static_cast<std::int64_t>(FLAGS_max_sessions) + |
| 52 | + FLAGS_prefix_cache_entries + (FLAGS_prefix_cache_entries > 0 ? 1 : 0); |
| 53 | + if (physical_sessions > std::numeric_limits<int>::max()) { |
| 54 | + std::cerr |
| 55 | + << "session and prefix-cache capacity exceeds executor row limit\n"; |
| 56 | + return 1; |
| 57 | + } |
| 58 | + // Model/backend diagnostics must not corrupt the JSONL output stream. |
| 59 | + const int protocol_output = dup(STDOUT_FILENO); |
| 60 | + if (protocol_output < 0 || dup2(STDERR_FILENO, STDOUT_FILENO) < 0) { |
| 61 | + if (protocol_output >= 0) |
| 62 | + close(protocol_output); |
| 63 | + return 1; |
| 64 | + } |
| 65 | + signal(SIGPIPE, SIG_IGN); |
| 66 | + int result = 1; |
| 67 | + try { |
| 68 | + namespace llm = executorch::extension::llm; |
| 69 | + auto tokenizer = llm::load_tokenizer(FLAGS_tokenizer); |
| 70 | + auto module = std::make_unique<executorch::extension::Module>(FLAGS_pte); |
| 71 | + if (!tokenizer || module->load() != executorch::runtime::Error::Ok) { |
| 72 | + throw std::runtime_error("could not load program or tokenizer"); |
| 73 | + } |
| 74 | + // Load PROGRAM only. ModuleExecutor binds cache before loading forward. |
| 75 | + const auto dtype = llm::read_activation_dtype(*module); |
| 76 | + const auto context = llm::read_max_context_length(*module); |
| 77 | + if (!dtype.ok() || !context.ok() || FLAGS_max_session_tokens > *context) { |
| 78 | + throw std::runtime_error( |
| 79 | + "invalid metadata or max_session_tokens exceeds model context"); |
| 80 | + } |
| 81 | + const auto eos = llm::get_eos_ids(tokenizer.get(), module.get()); |
| 82 | + auto executor = llm::batching::ModuleExecutor::create( |
| 83 | + std::move(module), |
| 84 | + static_cast<int>(physical_sessions), |
| 85 | + FLAGS_max_session_tokens, |
| 86 | + static_cast<int>(*dtype), |
| 87 | + -1, |
| 88 | + llm::cache::kind::kBatched); |
| 89 | + if (!executor.ok()) |
| 90 | + throw std::runtime_error("could not create packed ModuleExecutor"); |
| 91 | + const auto width = (*executor)->preferred_batch_tokens(); |
| 92 | + const auto decode_slots = |
| 93 | + static_cast<std::size_t>(FLAGS_max_decode_sequences); |
| 94 | + if (width == 0 || decode_slots >= width) { |
| 95 | + throw std::runtime_error( |
| 96 | + "packed forward width must exceed max_decode_sequences"); |
| 97 | + } |
| 98 | + auto scheduler = llm::batching::DecodeFirstScheduler::create( |
| 99 | + width, decode_slots, width - decode_slots); |
| 100 | + if (!scheduler) |
| 101 | + throw std::runtime_error("invalid scheduler limits"); |
| 102 | + llm::serving::ServingRuntimeConfig config; |
| 103 | + config.max_sessions = FLAGS_max_sessions; |
| 104 | + config.prefix_cache_capacity = FLAGS_prefix_cache_entries; |
| 105 | + config.max_context_length = FLAGS_max_session_tokens; |
| 106 | + config.max_requests = FLAGS_max_inflight_requests; |
| 107 | + config.max_pending_operations = FLAGS_max_inflight_requests; |
| 108 | + config.default_stop_tokens.assign(eos.begin(), eos.end()); |
| 109 | + llm::serving::ServingRuntime runtime( |
| 110 | + **executor, std::move(scheduler), *tokenizer, config); |
| 111 | + executorch::examples::llm_server::MultiplexedWorkerConfig transport; |
| 112 | + transport.max_inflight_requests = FLAGS_max_inflight_requests; |
| 113 | + result = executorch::examples::llm_server::run_multiplexed_worker( |
| 114 | + runtime, STDIN_FILENO, protocol_output, transport); |
| 115 | + } catch (const std::exception& error) { |
| 116 | + std::cerr << "worker startup failed: " << error.what() << '\n'; |
| 117 | + } |
| 118 | + close(protocol_output); |
| 119 | + return result; |
| 120 | +} |
0 commit comments