@@ -76,10 +76,9 @@ std::string LLM::generate(std::string input,
7676 return output;
7777}
7878
79- std::string LLM::generateMultimodal (
80- std::string prompt, std::shared_ptr<jsi::Function> callback,
81- std::vector<std::string> imagePaths, std::string imageToken,
82- std::vector<std::vector<float >> audioWaveforms, std::string audioToken) {
79+ std::string LLM::generateMultimodal (std::string prompt,
80+ std::shared_ptr<jsi::Function> callback,
81+ MultimodalInputs mutlimodalInputs) {
8382 if (!runner_ || !runner_->is_loaded ()) {
8483 throw RnExecutorchError (RnExecutorchErrorCode::ModuleNotLoaded,
8584 " Runner is not loaded" );
@@ -88,7 +87,8 @@ std::string LLM::generateMultimodal(
8887 throw RnExecutorchError (RnExecutorchErrorCode::InvalidUserInput,
8988 " This model does not support multimodal input." );
9089 }
91- if (imageToken.empty () && audioToken.empty ()) {
90+ if (mutlimodalInputs.imageToken .empty () &&
91+ mutlimodalInputs.audioToken .empty ()) {
9292 throw RnExecutorchError (
9393 RnExecutorchErrorCode::InvalidUserInput,
9494 " At least one of imageToken/audioToken must be non-empty" );
@@ -98,12 +98,13 @@ std::string LLM::generateMultimodal(
9898 // so that image/audio placeholders can be freely interleaved in the prompt.
9999 std::vector<llm::MultimodalInput> inputs;
100100 size_t imageIdx = 0 , audioIdx = 0 , pos = 0 ;
101- constexpr int32_t kAudioSampleRate = 16000 ;
102101 while (pos < prompt.size ()) {
103- size_t imgAt =
104- imageToken.empty () ? std::string::npos : prompt.find (imageToken, pos);
105- size_t audAt =
106- audioToken.empty () ? std::string::npos : prompt.find (audioToken, pos);
102+ size_t imgAt = mutlimodalInputs.imageToken .empty ()
103+ ? std::string::npos
104+ : prompt.find (mutlimodalInputs.imageToken , pos);
105+ size_t audAt = mutlimodalInputs.audioToken .empty ()
106+ ? std::string::npos
107+ : prompt.find (mutlimodalInputs.audioToken , pos);
107108 if (imgAt == std::string::npos && audAt == std::string::npos) {
108109 inputs.push_back (llm::make_text_input (prompt.substr (pos)));
109110 break ;
@@ -115,25 +116,27 @@ std::string LLM::generateMultimodal(
115116 inputs.push_back (llm::make_text_input (prompt.substr (pos, at - pos)));
116117 }
117118 if (imageFirst) {
118- if (imageIdx >= imagePaths.size ()) {
119+ if (imageIdx >= mutlimodalInputs. imagePaths .size ()) {
119120 throw RnExecutorchError (RnExecutorchErrorCode::InvalidUserInput,
120- " More '" + imageToken +
121+ " More '" + mutlimodalInputs. imageToken +
121122 " ' placeholders than image paths" );
122123 }
123- inputs.push_back (llm::make_image_input (imagePaths[imageIdx++]));
124- pos = at + imageToken.size ();
124+ inputs.push_back (
125+ llm::make_image_input (mutlimodalInputs.imagePaths [imageIdx++]));
126+ pos = at + mutlimodalInputs.imageToken .size ();
125127 } else {
126- if (audioIdx >= audioWaveforms.size ()) {
128+ if (audioIdx >= mutlimodalInputs. audioWaveforms .size ()) {
127129 throw RnExecutorchError (RnExecutorchErrorCode::InvalidUserInput,
128- " More '" + audioToken +
130+ " More '" + mutlimodalInputs. audioToken +
129131 " ' placeholders than audio waveforms" );
130132 }
131133 inputs.push_back (llm::make_audio_input (
132- std::move (audioWaveforms[audioIdx++]), kAudioSampleRate ));
133- pos = at + audioToken.size ();
134+ std::move (mutlimodalInputs. audioWaveforms [audioIdx++])));
135+ pos = at + mutlimodalInputs. audioToken .size ();
134136 }
135137 }
136- if (imageIdx < imagePaths.size () || audioIdx < audioWaveforms.size ()) {
138+ if (imageIdx < mutlimodalInputs.imagePaths .size () ||
139+ audioIdx < mutlimodalInputs.audioWaveforms .size ()) {
137140 throw RnExecutorchError (
138141 RnExecutorchErrorCode::InvalidUserInput,
139142 " More image/audio paths provided than placeholders in prompt" );
0 commit comments