Skip to content

Commit 128e126

Browse files
authored
[LLM serving 3/x] Bridge serving requests to the batching runner (pytorch#23284)
Add bounded asynchronous generation on top of the serving session lifecycle. Requests receive an immediate, cancellable RequestHandle and execute through the batching runner using either a named session or an independent ephemeral session. * Deliver output off the engine thread through bounded per-request mailboxes and one shared delivery thread. Native sinks must perform short, nonblocking work. * Add an optional batching settlement notification so delivery can resume once the final outcome and metrics are published, without polling or waiting on individual generations. * Preserve completion ordering: deliver tokens, prepare completion, finalize session state on control, then deliver the terminal event and release request admission. * Handle overload, cancellation, output overflow, callback failures, and shutdown. Fence session ownership so stale requests cannot modify a session replaced by close/reset. The token-delta submission bridge remains internal. Public prompt preparation and text generation follow in PR4.
1 parent 9f975b8 commit 128e126

59 files changed

Lines changed: 17861 additions & 265 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/_llm_server.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ jobs:
5656
cmake -S examples/llm_server/cpp -B cmake-out/examples/llm_server/cpp \
5757
-DCMAKE_PREFIX_PATH="${PWD}/cmake-out"
5858
cmake --build cmake-out/examples/llm_server/cpp \
59-
--target test_worker_prefill_plan test_worker_loop \
59+
--target test_worker_prefill_plan test_worker_loop test_multiplexed_worker test_batching_worker \
6060
-j "$(nproc)"
6161
ctest --test-dir cmake-out/examples/llm_server/cpp --output-on-failure
62+
63+
export EXECUTORCH_BATCHING_TEST_WORKER="${PWD}/cmake-out/examples/llm_server/cpp/test_batching_worker"
64+
test -x "${EXECUTORCH_BATCHING_TEST_WORKER}"
65+
python -m pytest -q examples/llm_server/python/tests/test_batching_end_to_end.py

‎.github/workflows/pull.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,8 @@ jobs:
760760
contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer/serving') ||
761761
contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer/tests') ||
762762
contains(needs.changed-files.outputs.changed-files, 'examples/models/muse_glimmer') ||
763+
contains(needs.changed-files.outputs.changed-files, 'extension/llm/batching') ||
764+
contains(needs.changed-files.outputs.changed-files, 'extension/llm/serving') ||
763765
contains(needs.changed-files.outputs.changed-files, 'extension/llm/runner') ||
764766
contains(needs.changed-files.outputs.changed-files, 'extension/llm/tokenizers') ||
765767
contains(needs.changed-files.outputs.changed-files, '.github/workflows/_get-changed-files.yml') ||

‎.github/workflows/trunk.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,6 +1000,8 @@ jobs:
10001000
contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer/serving') ||
10011001
contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer/tests') ||
10021002
contains(needs.changed-files.outputs.changed-files, 'examples/models/muse_glimmer') ||
1003+
contains(needs.changed-files.outputs.changed-files, 'extension/llm/batching') ||
1004+
contains(needs.changed-files.outputs.changed-files, 'extension/llm/serving') ||
10031005
contains(needs.changed-files.outputs.changed-files, 'extension/llm/runner') ||
10041006
contains(needs.changed-files.outputs.changed-files, 'extension/llm/tokenizers') ||
10051007
contains(needs.changed-files.outputs.changed-files, '.github/workflows/_get-changed-files.yml') ||

‎backends/mlx/examples/llm/CMakeLists.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,20 @@ endif()
8383
set(EXECUTORCH_BUILD_MLX ON)
8484
executorch_target_copy_mlx_metallib(mlx_run_llm_hf)
8585
executorch_target_copy_mlx_metallib(mlx_run_llm_batched)
86+
87+
# Shared JSONL adapter over the packed ModuleExecutor, not the legacy worker.
88+
find_package(Threads REQUIRED)
89+
set(_worker_dir ${EXECUTORCH_ROOT}/examples/llm_server/cpp)
90+
add_executable(
91+
llm_worker ${_worker_dir}/module_worker_main.cpp
92+
${_worker_dir}/multiplexed_worker.cpp
93+
)
94+
target_include_directories(
95+
llm_worker PRIVATE ${_common_include_directories} ${_json_include}
96+
${_flatbuffers_include}
97+
)
98+
target_link_libraries(
99+
llm_worker PRIVATE ${link_libraries} extension_llm_batching_module
100+
extension_llm_serving Threads::Threads
101+
)
102+
executorch_target_copy_mlx_metallib(llm_worker)

‎examples/llm_server/cpp/BUCK‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,14 @@ fbcode_target(_kind = runtime.cxx_library,
1919
external_deps = ["nlohmann_json"],
2020
visibility = ["PUBLIC"],
2121
)
22+
23+
fbcode_target(_kind = runtime.cxx_library,
24+
name = "multiplexed_worker",
25+
srcs = ["multiplexed_worker.cpp"],
26+
headers = ["multiplexed_worker_test.h"],
27+
exported_headers = ["multiplexed_worker.h"],
28+
header_namespace = "executorch/examples/llm_server/cpp",
29+
exported_deps = ["//executorch/extension/llm/serving:serving_runtime"],
30+
external_deps = ["nlohmann_json"],
31+
visibility = ["PUBLIC"],
32+
)

‎examples/llm_server/cpp/CMakeLists.txt‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,19 @@ set(_json_include
2525

2626
# executorch
2727
list(APPEND CMAKE_FIND_ROOT_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../..)
28-
find_package(executorch CONFIG REQUIRED FIND_ROOT_PATH_BOTH)
28+
if(NOT TARGET executorch)
29+
find_package(executorch CONFIG REQUIRED FIND_ROOT_PATH_BOTH)
30+
# Older tokenizer exports expose their internal headers as top-level includes,
31+
# shadowing the POSIX regex.h used by GoogleTest. Use the public include root.
32+
get_target_property(
33+
_tokenizer_includes tokenizers::tokenizers INTERFACE_INCLUDE_DIRECTORIES
34+
)
35+
list(TRANSFORM _tokenizer_includes REPLACE "/pytorch/tokenizers/?$" "")
36+
set_target_properties(
37+
tokenizers::tokenizers PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
38+
"${_tokenizer_includes}"
39+
)
40+
endif()
2941
find_package(Threads REQUIRED)
3042
executorch_target_link_options_shared_lib(executorch)
3143

@@ -57,3 +69,54 @@ target_link_libraries(
5769
test_worker_loop PUBLIC ${test_link_libraries} Threads::Threads
5870
)
5971
add_test(NAME worker_loop COMMAND test_worker_loop)
72+
73+
# New transport is deliberately separate from the legacy worker loop.
74+
add_library(multiplexed_worker STATIC multiplexed_worker.cpp)
75+
target_include_directories(
76+
multiplexed_worker
77+
PUBLIC ${_common_include_directories}
78+
PRIVATE ${_json_include}
79+
)
80+
target_link_libraries(
81+
multiplexed_worker PUBLIC extension_llm_serving Threads::Threads
82+
)
83+
84+
if(NOT TARGET GTest::gtest)
85+
find_package(GTest CONFIG QUIET)
86+
if(NOT TARGET GTest::gtest)
87+
add_subdirectory(
88+
${EXECUTORCH_ROOT}/third-party/googletest
89+
${CMAKE_CURRENT_BINARY_DIR}/googletest EXCLUDE_FROM_ALL
90+
)
91+
endif()
92+
endif()
93+
add_executable(test_multiplexed_worker test/test_multiplexed_worker.cpp)
94+
target_include_directories(test_multiplexed_worker PRIVATE ${_json_include})
95+
target_link_libraries(
96+
test_multiplexed_worker PRIVATE multiplexed_worker GTest::gtest
97+
)
98+
add_test(NAME multiplexed_worker COMMAND test_multiplexed_worker)
99+
100+
option(
101+
EXECUTORCH_LLM_SERVER_PYTHON_TESTS
102+
"Run native-backed Python client tests (requires Python test dependencies)"
103+
OFF
104+
)
105+
if(EXECUTORCH_LLM_SERVER_PYTHON_TESTS)
106+
find_package(Python3 REQUIRED COMPONENTS Interpreter)
107+
set(_native_worker $<TARGET_FILE:test_multiplexed_worker>)
108+
set(_python_tests ${CMAKE_CURRENT_SOURCE_DIR}/../python/tests)
109+
add_test(
110+
NAME multiplexed_worker_client_native
111+
COMMAND
112+
${CMAKE_COMMAND} -E env
113+
"EXECUTORCH_TEST_MULTIPLEXED_WORKER=${_native_worker}"
114+
"PYTHONPATH=${EXECUTORCH_ROOT}/.." ${Python3_EXECUTABLE} -m pytest -q
115+
${_python_tests}/test_multiplexed_worker_client_native.py
116+
)
117+
set_tests_properties(multiplexed_worker_client_native PROPERTIES TIMEOUT 120)
118+
endif()
119+
120+
# Native subprocess fixture for the opt-in Python/OpenAI integration tests.
121+
add_executable(test_batching_worker test_batching_worker_main.cpp)
122+
target_link_libraries(test_batching_worker PRIVATE multiplexed_worker)
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <unistd.h>
10+
#include <csignal>
11+
#include <cstdint>
12+
#include <iostream>
13+
#include <limits>
14+
#include <memory>
15+
#include <stdexcept>
16+
17+
#include <gflags/gflags.h>
18+
19+
#include <executorch/examples/llm_server/cpp/multiplexed_worker.h>
20+
#include <executorch/extension/llm/batching/decode_first_scheduler.h>
21+
#include <executorch/extension/llm/batching/module_executor.h>
22+
#include <executorch/extension/llm/cache/cache_registry.h>
23+
#include <executorch/extension/llm/runner/llm_runner_helper.h>
24+
#include <executorch/extension/llm/runner/model_metadata.h>
25+
#include <executorch/extension/module/module.h>
26+
27+
DEFINE_string(pte, "", "Packed off-graph-cache text model program");
28+
DEFINE_string(tokenizer, "", "Tokenizer matching the program");
29+
DEFINE_int32(max_sessions, 4, "Maximum logical serving sessions");
30+
DEFINE_int32(
31+
prefix_cache_entries,
32+
0,
33+
"Greedy creation-only prompt snapshots; zero disables prefix caching");
34+
DEFINE_int32(max_session_tokens, 2048, "Maximum retained tokens per session");
35+
DEFINE_int32(max_decode_sequences, 2, "Decode slots in one packed forward");
36+
DEFINE_int32(
37+
max_inflight_requests,
38+
64,
39+
"Bounded wire generation/lifecycle capacity");
40+
41+
int main(int argc, char** argv) {
42+
gflags::ParseCommandLineFlags(&argc, &argv, true);
43+
if (FLAGS_pte.empty() || FLAGS_tokenizer.empty() || FLAGS_max_sessions <= 0 ||
44+
FLAGS_max_session_tokens <= 0 || FLAGS_max_decode_sequences <= 0 ||
45+
FLAGS_max_inflight_requests <= 0 || FLAGS_prefix_cache_entries < 0) {
46+
std::cerr
47+
<< "--pte and --tokenizer are required; capacity limits must be positive "
48+
"and prefix_cache_entries nonnegative\n";
49+
return 1;
50+
}
51+
const auto physical_sessions = static_cast<std::int64_t>(FLAGS_max_sessions) +
52+
FLAGS_prefix_cache_entries + (FLAGS_prefix_cache_entries > 0 ? 1 : 0);
53+
if (physical_sessions > std::numeric_limits<int>::max()) {
54+
std::cerr
55+
<< "session and prefix-cache capacity exceeds executor row limit\n";
56+
return 1;
57+
}
58+
// Model/backend diagnostics must not corrupt the JSONL output stream.
59+
const int protocol_output = dup(STDOUT_FILENO);
60+
if (protocol_output < 0 || dup2(STDERR_FILENO, STDOUT_FILENO) < 0) {
61+
if (protocol_output >= 0)
62+
close(protocol_output);
63+
return 1;
64+
}
65+
signal(SIGPIPE, SIG_IGN);
66+
int result = 1;
67+
try {
68+
namespace llm = executorch::extension::llm;
69+
auto tokenizer = llm::load_tokenizer(FLAGS_tokenizer);
70+
auto module = std::make_unique<executorch::extension::Module>(FLAGS_pte);
71+
if (!tokenizer || module->load() != executorch::runtime::Error::Ok) {
72+
throw std::runtime_error("could not load program or tokenizer");
73+
}
74+
// Load PROGRAM only. ModuleExecutor binds cache before loading forward.
75+
const auto dtype = llm::read_activation_dtype(*module);
76+
const auto context = llm::read_max_context_length(*module);
77+
if (!dtype.ok() || !context.ok() || FLAGS_max_session_tokens > *context) {
78+
throw std::runtime_error(
79+
"invalid metadata or max_session_tokens exceeds model context");
80+
}
81+
const auto eos = llm::get_eos_ids(tokenizer.get(), module.get());
82+
auto executor = llm::batching::ModuleExecutor::create(
83+
std::move(module),
84+
static_cast<int>(physical_sessions),
85+
FLAGS_max_session_tokens,
86+
static_cast<int>(*dtype),
87+
-1,
88+
llm::cache::kind::kBatched);
89+
if (!executor.ok())
90+
throw std::runtime_error("could not create packed ModuleExecutor");
91+
const auto width = (*executor)->preferred_batch_tokens();
92+
const auto decode_slots =
93+
static_cast<std::size_t>(FLAGS_max_decode_sequences);
94+
if (width == 0 || decode_slots >= width) {
95+
throw std::runtime_error(
96+
"packed forward width must exceed max_decode_sequences");
97+
}
98+
auto scheduler = llm::batching::DecodeFirstScheduler::create(
99+
width, decode_slots, width - decode_slots);
100+
if (!scheduler)
101+
throw std::runtime_error("invalid scheduler limits");
102+
llm::serving::ServingRuntimeConfig config;
103+
config.max_sessions = FLAGS_max_sessions;
104+
config.prefix_cache_capacity = FLAGS_prefix_cache_entries;
105+
config.max_context_length = FLAGS_max_session_tokens;
106+
config.max_requests = FLAGS_max_inflight_requests;
107+
config.max_pending_operations = FLAGS_max_inflight_requests;
108+
config.default_stop_tokens.assign(eos.begin(), eos.end());
109+
llm::serving::ServingRuntime runtime(
110+
**executor, std::move(scheduler), *tokenizer, config);
111+
executorch::examples::llm_server::MultiplexedWorkerConfig transport;
112+
transport.max_inflight_requests = FLAGS_max_inflight_requests;
113+
result = executorch::examples::llm_server::run_multiplexed_worker(
114+
runtime, STDIN_FILENO, protocol_output, transport);
115+
} catch (const std::exception& error) {
116+
std::cerr << "worker startup failed: " << error.what() << '\n';
117+
}
118+
close(protocol_output);
119+
return result;
120+
}

0 commit comments

Comments
 (0)