1515#include < condition_variable>
1616#include < cstddef>
1717#include < exception>
18+ #include < iterator>
1819#include < limits>
1920#include < mutex>
2021#include < thread>
@@ -302,7 +303,7 @@ class RunnerImpl : public std::enable_shared_from_this<RunnerImpl> {
302303 void request_close (SessionId session) noexcept ;
303304 GenerationHandle generate_async (
304305 SessionId session,
305- std::vector<Token> delta,
306+ GenerationInput delta,
306307 GenConfig config,
307308 GenerationCallback on_update,
308309 std::function<void ()> on_settled);
@@ -337,7 +338,8 @@ class RunnerImpl : public std::enable_shared_from_this<RunnerImpl> {
337338 // admission. Scheduler tasks own the delta after submission.
338339 struct GenerationRequest {
339340 SessionId session = 0 ;
340- std::shared_ptr<const std::vector<Token>> delta;
341+ InputPayload delta;
342+ std::size_t size = 0 ;
341343 SamplingParams sampling;
342344 std::optional<std::uint64_t > seed;
343345 Generation generation;
@@ -453,21 +455,24 @@ class RunnerImpl : public std::enable_shared_from_this<RunnerImpl> {
453455 std::optional<TerminalOutcome> validate_generation_start_ (
454456 const GenerationRequest& request,
455457 const SessionRecord& record) const ;
456- std::shared_ptr<const std::vector<Token>> build_initial_delta_ (
458+ // Merge a pending token into raw input, or return it as a separate prefix.
459+ std::optional<Token> carry_pending_ (
457460 GenerationRequest& request,
458461 const SessionRecord& record) const ;
459462
460- // Split `tokens ` into chunks, the last of which produces output.
463+ // Split `payload ` into chunks, the last of which may produce output.
461464 //
462465 // `is_continuation` distinguishes what the executor handed back from the
463466 // caller's opening delta. Only a continuation of exactly one token is a
464467 // decode: an opening delta is prefill however short it is, and a longer
465468 // continuation is re-fed as prefill too.
466469 std::vector<Task> create_tasks_ (
467470 SessionId session,
468- std::shared_ptr<const std::vector<Token>> tokens,
471+ InputPayload payload,
472+ std::size_t size,
469473 Position position,
470- bool is_continuation);
474+ bool is_continuation,
475+ bool produce_output = true );
471476 InterpretedOutput interpret_output_ (
472477 const Generation& generation,
473478 const Output& output) const ;
@@ -619,7 +624,7 @@ Session::make_clone_request(Position upto) const {
619624}
620625
621626GenerationHandle Session::generate_async (
622- std::vector<Token> delta,
627+ GenerationInput delta,
623628 GenConfig config,
624629 GenerationCallback on_update,
625630 std::function<void ()> on_settled) const {
@@ -1307,15 +1312,22 @@ void RunnerImpl::request_close(SessionId session) noexcept {
13071312
13081313GenerationHandle RunnerImpl::generate_async (
13091314 SessionId session,
1310- std::vector<Token> delta,
1315+ GenerationInput delta,
13111316 GenConfig config,
13121317 GenerationCallback on_update,
13131318 std::function<void ()> on_settled) {
13141319 auto state = std::make_shared<GenerationHandleState>();
13151320 state->on_settled = std::move (on_settled);
13161321 GenerationRequest request;
13171322 request.session = session;
1318- request.delta = std::make_shared<const std::vector<Token>>(std::move (delta));
1323+ request.delta = std::visit (
1324+ Overloaded{
1325+ [](std::vector<Token> tokens) -> InputPayload {
1326+ return std::make_shared<const std::vector<Token>>(
1327+ std::move (tokens));
1328+ },
1329+ [](PreparedInputPtr prepared) -> InputPayload { return prepared; }},
1330+ std::move (delta));
13191331 request.seed = config.seed ;
13201332 request.sampling = std::move (config.sampling );
13211333 request.generation .remaining_tokens = config.max_new_tokens ;
@@ -1353,7 +1365,7 @@ std::optional<TerminalOutcome> RunnerImpl::validate_generation_start_(
13531365 if (request.generation .remaining_tokens <= 0 ) {
13541366 return TerminalOutcome::failed (" max_new_tokens must be greater than zero" );
13551367 }
1356- if (! request.delta || request. delta -> empty () ) {
1368+ if (request.size == 0 ) {
13571369 return TerminalOutcome::failed (" generation delta must not be empty" );
13581370 }
13591371
@@ -1364,7 +1376,7 @@ std::optional<TerminalOutcome> RunnerImpl::validate_generation_start_(
13641376 const auto room = static_cast <std::size_t >(
13651377 std::numeric_limits<Position>::max () - start_position);
13661378 const auto pending_tokens = record.pending ? 1u : 0u ;
1367- if (pending_tokens > room || request.delta -> size () > room - pending_tokens) {
1379+ if (pending_tokens > room || request.size > room - pending_tokens) {
13681380 return TerminalOutcome::failed (
13691381 " generation delta exceeds the session position range" );
13701382 }
@@ -1375,22 +1387,33 @@ std::optional<TerminalOutcome> RunnerImpl::validate_generation_start_(
13751387 if (record.active_generation ) {
13761388 return TerminalOutcome::failed (" session already has an active generation" );
13771389 }
1390+ if (const auto * prepared = std::get_if<PreparedInputPtr>(&request.delta );
1391+ prepared && !executor_.accepts (**prepared)) {
1392+ return TerminalOutcome::failed (" executor does not accept prepared input" );
1393+ }
13781394 return std::nullopt ;
13791395}
13801396
1381- std::shared_ptr< const std::vector< Token>> RunnerImpl::build_initial_delta_ (
1397+ std::optional< Token> RunnerImpl::carry_pending_ (
13821398 GenerationRequest& request,
13831399 const SessionRecord& record) const {
1384- auto delta = std::move (request.delta );
13851400 if (!record.pending ) {
1386- return delta;
1401+ return std::nullopt ;
1402+ }
1403+ const auto * tokens = std::get_if<TokenInputPtr>(&request.delta );
1404+ if (!tokens) {
1405+ // Opaque backing cannot be prepended to: send a separate raw prefill.
1406+ return record.pending ;
13871407 }
13881408
13891409 std::vector<Token> carried;
1390- carried.reserve (delta ->size () + 1 );
1410+ carried.reserve ((*tokens) ->size () + 1 );
13911411 carried.push_back (*record.pending );
1392- carried.insert (carried.end (), delta->begin (), delta->end ());
1393- return std::make_shared<const std::vector<Token>>(std::move (carried));
1412+ carried.insert (carried.end (), (*tokens)->begin (), (*tokens)->end ());
1413+ ++request.size ;
1414+ request.delta =
1415+ std::make_shared<const std::vector<Token>>(std::move (carried));
1416+ return std::nullopt ;
13941417}
13951418
13961419void RunnerImpl::start_generation_ (GenerationRequest request) {
@@ -1415,20 +1438,23 @@ void RunnerImpl::start_generation_(GenerationRequest request) {
14151438 return ;
14161439 }
14171440 auto & record = session->second ;
1441+ request.size = std::visit (
1442+ [](const auto & payload) -> std::size_t {
1443+ return payload ? payload->size () : 0 ;
1444+ },
1445+ request.delta );
14181446 if (auto rejection = validate_generation_start_ (request, record)) {
14191447 complete_request_ (
14201448 std::move (request), std::move (*rejection), /* on_engine_thread=*/ true );
14211449 return ;
14221450 }
14231451
1424- const auto start_position = record.position ();
1425- // The caller's own delta, captured before build_initial_delta_ moves it and
1426- // before any carried token is prepended. The generation tier counts what
1427- // callers gave; tokens actually fed to the model are EngineMetrics'
1428- // model_input_tokens().
1452+ auto start_position = record.position ();
1453+ // Count the caller's own delta before adding a carried token. EngineMetrics'
1454+ // model_input_tokens() counts all positions actually fed to the model.
14291455 request.generation .m .n_prompt_tokens =
1430- static_cast <std::int64_t >(request.delta -> size () );
1431- auto delta = build_initial_delta_ (request, record);
1456+ static_cast <std::int64_t >(request.size );
1457+ const auto prefix = carry_pending_ (request, record);
14321458 if (!is_running_ () || !record.status ->open .load (std::memory_order_acquire) ||
14331459 request.generation .state ->cancelled .load ()) {
14341460 complete_request_ (
@@ -1458,22 +1484,38 @@ void RunnerImpl::start_generation_(GenerationRequest request) {
14581484 return ;
14591485 }
14601486
1461- // advance() clears a carried token only after the input holding it runs.
1462- submit_ (
1487+ std::vector<Task> tasks;
1488+ if (prefix) {
1489+ tasks = create_tasks_ (
1490+ request.session ,
1491+ std::make_shared<const std::vector<Token>>(1 , *prefix),
1492+ 1 ,
1493+ start_position++,
1494+ /* is_continuation=*/ false ,
1495+ /* produce_output=*/ false );
1496+ }
1497+ auto suffix = create_tasks_ (
14631498 request.session ,
1464- create_tasks_ (
1465- request.session ,
1466- std::move (delta),
1467- start_position,
1468- /* is_continuation=*/ false ));
1499+ std::move (request.delta ),
1500+ request.size ,
1501+ start_position,
1502+ /* is_continuation=*/ false );
1503+ tasks.insert (
1504+ tasks.end (),
1505+ std::make_move_iterator (suffix.begin ()),
1506+ std::make_move_iterator (suffix.end ()));
1507+ // advance() clears a carried token only after the input holding it runs.
1508+ submit_ (request.session , std::move (tasks));
14691509}
14701510
14711511std::vector<Task> RunnerImpl::create_tasks_ (
14721512 SessionId session,
1473- std::shared_ptr<const std::vector<Token>> tokens,
1513+ InputPayload payload,
1514+ std::size_t size,
14741515 Position position,
1475- bool is_continuation) {
1476- const auto total = static_cast <std::int32_t >(tokens->size ());
1516+ bool is_continuation,
1517+ bool produce_output) {
1518+ const auto total = static_cast <std::int32_t >(size);
14771519 // The scheduler owns this limit, so the runner cannot split a prompt into
14781520 // chunks the scheduler would then refuse. It is non-zero and clamped to the
14791521 // token count, so the loop below always advances.
@@ -1495,10 +1537,10 @@ std::vector<Task> RunnerImpl::create_tasks_(
14951537 t.cancelled = false ;
14961538 t.is_decode = decode;
14971539 t.input .sid = session;
1498- t.input .produce_output = last;
1540+ t.input .produce_output = produce_output && last;
14991541 t.input .offset = static_cast <size_t >(i);
15001542 t.input .size = static_cast <size_t >(n);
1501- t.input .tokens = tokens ;
1543+ t.input .payload = payload ;
15021544 t.input .position = position;
15031545 tasks.push_back (std::move (t));
15041546 }
@@ -1707,6 +1749,7 @@ void RunnerImpl::resume_generation_(
17071749 create_tasks_ (
17081750 session_id,
17091751 std::move (input),
1752+ 1 ,
17101753 position,
17111754 /* is_continuation=*/ true ));
17121755}
0 commit comments