Skip to content

Commit 467cc0a

Browse files
mkopcinsmkopcins
committed
added tests
1 parent 71ba63c commit 467cc0a

4 files changed

Lines changed: 147 additions & 18 deletions

File tree

‎apps/llm/app/multimodal_llm/index.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ function MultimodalLLMScreen() {
244244
<Text style={styles.helloText}>Hello! 👋</Text>
245245
<Text style={styles.bottomHelloText}>
246246
{model.capabilities.find((c) => c === 'audio')
247-
? 'Say hi or pick an image or and ask me anything about it.'
247+
? 'Say "Hi" or pick an image or and ask me anything about it.'
248248
: 'Pick an image and ask me anything about it.'}
249249
</Text>
250250
<SuggestedPrompts

‎packages/react-native-executorch/common/rnexecutorch/tests/integration/LLMTest.cpp‎

Lines changed: 133 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
#include "BaseModelTests.h"
2+
#include "utils/TestUtils.h"
23
#include <gtest/gtest.h>
34
#include <memory>
45
#include <string>
6+
#include <utility>
7+
#include <vector>
58

69
#include <ReactCommon/CallInvoker.h>
710
#include <rnexecutorch/Error.h>
811
#include <rnexecutorch/models/llm/LLM.h>
12+
#include <runner/encoders/audio_encoder.h>
913
#include <runner/encoders/vision_encoder.h>
1014

1115
using namespace rnexecutorch;
@@ -30,6 +34,12 @@ std::string formatChatML(const std::string &systemPrompt,
3034
"<|im_start|>assistant\n";
3135
}
3236

37+
// Helper to format a single-turn prompt in Gemma's chat template.
38+
std::string formatGemma(const std::string &userMessage) {
39+
return "<start_of_turn>user\n" + userMessage + "<end_of_turn>\n" +
40+
"<start_of_turn>model\n";
41+
}
42+
3343
// ============================================================================
3444
// Common tests via typed test suite
3545
// ============================================================================
@@ -227,6 +237,18 @@ TEST(VisionEncoderTest, LoadFailsWithClearErrorWhenMethodMissing) {
227237
EXPECT_THROW(encoder->load(), rnexecutorch::RnExecutorchError);
228238
}
229239

240+
TEST(AudioEncoderTest, LoadFailsWithClearErrorWhenMethodMissing) {
241+
// smolLm2_135M_8da4w.pte has no audio_encoder method
242+
auto module = std::make_unique<::executorch::extension::Module>(
243+
"smolLm2_135M_8da4w.pte",
244+
::executorch::extension::Module::LoadMode::File);
245+
246+
auto encoder =
247+
std::make_unique<executorch::extension::llm::AudioEncoder>(*module);
248+
249+
EXPECT_THROW(encoder->load(), rnexecutorch::RnExecutorchError);
250+
}
251+
230252
// ============================================================================
231253
// VLM-specific tests
232254
// ============================================================================
@@ -243,7 +265,11 @@ TEST_F(LLMTest, TextModelIsNotMultimodal) {
243265

244266
TEST_F(LLMTest, GenerateMultimodalOnTextModelThrows) {
245267
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
246-
EXPECT_THROW(model.generateMultimodal("hello", {}, "<image>", nullptr),
268+
// A text-only runner reports is_multimodal() == false, so any multimodal
269+
// call must be rejected before the inputs are even inspected.
270+
MultimodalInputs inputs{.images =
271+
ImageInputs{.paths = {}, .token = "<image>"}};
272+
EXPECT_THROW(model.generateMultimodal("hello", nullptr, std::move(inputs)),
247273
RnExecutorchError);
248274
}
249275

@@ -270,22 +296,120 @@ std::shared_ptr<facebook::react::CallInvoker> VLMTest::invoker_;
270296
std::unique_ptr<LLM> VLMTest::model_;
271297

272298
TEST_F(VLMTest, GenerateMultimodalEmptyImageTokenThrows) {
273-
EXPECT_THROW(
274-
model_->generateMultimodal("hello", {kTestImagePath}, "", nullptr),
275-
RnExecutorchError);
299+
MultimodalInputs inputs{
300+
.images = ImageInputs{.paths = {kTestImagePath}, .token = ""}};
301+
EXPECT_THROW(model_->generateMultimodal("hello", nullptr, std::move(inputs)),
302+
RnExecutorchError);
276303
}
277304

278305
TEST_F(VLMTest, GenerateMultimodalMorePlaceholdersThanImagePaths) {
279306
std::string prompt = std::string(kVlmImageToken) + " and " + kVlmImageToken;
280-
EXPECT_THROW(model_->generateMultimodal(prompt, {kTestImagePath},
281-
kVlmImageToken, nullptr),
307+
MultimodalInputs inputs{.images = ImageInputs{.paths = {kTestImagePath},
308+
.token = kVlmImageToken}};
309+
EXPECT_THROW(model_->generateMultimodal(prompt, nullptr, std::move(inputs)),
282310
RnExecutorchError);
283311
}
284312

285313
TEST_F(VLMTest, GenerateMultimodalMoreImagePathsThanPlaceholders) {
286314
std::string prompt = std::string(kVlmImageToken) + " describe";
287-
EXPECT_THROW(model_->generateMultimodal(prompt,
288-
{kTestImagePath, kTestImagePath},
289-
kVlmImageToken, nullptr),
315+
MultimodalInputs inputs{
316+
.images = ImageInputs{.paths = {kTestImagePath, kTestImagePath},
317+
.token = kVlmImageToken}};
318+
EXPECT_THROW(model_->generateMultimodal(prompt, nullptr, std::move(inputs)),
319+
RnExecutorchError);
320+
}
321+
322+
// ============================================================================
323+
// Audio (Gemma 4) multimodal tests
324+
// ============================================================================
325+
constexpr auto kGemmaModelPath = "gemma4_e2b_mm_xnnpack.pte";
326+
constexpr auto kGemmaTokenizerPath = "gemma_tokenizer.json";
327+
constexpr auto kGemmaAudioToken = "<audio_soft_token>";
328+
constexpr auto kTestAudioPath = "test_audio_float.raw";
329+
330+
// Fixture that loads the audio-capable Gemma model once for all audio tests.
331+
class GemmaAudioTest : public ::testing::Test {
332+
protected:
333+
static void SetUpTestSuite() {
334+
invoker_ = createMockCallInvoker();
335+
model_ = std::make_unique<LLM>(kGemmaModelPath, kGemmaTokenizerPath,
336+
std::vector<std::string>{"vision", "audio"},
337+
invoker_);
338+
}
339+
340+
static void TearDownTestSuite() {
341+
model_.reset();
342+
invoker_.reset();
343+
}
344+
345+
static std::vector<float> loadAudio(size_t maxSamples = 32000) {
346+
auto wav = test_utils::loadAudioFromFile(kTestAudioPath);
347+
if (wav.size() > maxSamples) {
348+
wav.resize(maxSamples);
349+
}
350+
return wav;
351+
}
352+
353+
static std::shared_ptr<facebook::react::CallInvoker> invoker_;
354+
static std::unique_ptr<LLM> model_;
355+
};
356+
357+
std::shared_ptr<facebook::react::CallInvoker> GemmaAudioTest::invoker_;
358+
std::unique_ptr<LLM> GemmaAudioTest::model_;
359+
360+
TEST_F(GemmaAudioTest, GenerateMultimodalNoInputsThrows) {
361+
EXPECT_THROW(model_->generateMultimodal("hello", nullptr, {}),
362+
RnExecutorchError);
363+
}
364+
365+
TEST_F(GemmaAudioTest, GenerateMultimodalEmptyAudioTokenThrows) {
366+
MultimodalInputs inputs{
367+
.audios = AudioInputs{.waveforms = {loadAudio()}, .token = ""}};
368+
EXPECT_THROW(model_->generateMultimodal("hello", nullptr, std::move(inputs)),
369+
RnExecutorchError);
370+
}
371+
372+
TEST_F(GemmaAudioTest, GenerateMultimodalMorePlaceholdersThanWaveformsThrows) {
373+
std::string prompt =
374+
std::string(kGemmaAudioToken) + " and " + kGemmaAudioToken;
375+
MultimodalInputs inputs{.audios = AudioInputs{.waveforms = {loadAudio()},
376+
.token = kGemmaAudioToken}};
377+
EXPECT_THROW(model_->generateMultimodal(prompt, nullptr, std::move(inputs)),
290378
RnExecutorchError);
291379
}
380+
381+
TEST_F(GemmaAudioTest, GenerateMultimodalMoreWaveformsThanPlaceholdersThrows) {
382+
std::string prompt = std::string(kGemmaAudioToken) + " describe";
383+
MultimodalInputs inputs{
384+
.audios = AudioInputs{.waveforms = {loadAudio(), loadAudio()},
385+
.token = kGemmaAudioToken}};
386+
EXPECT_THROW(model_->generateMultimodal(prompt, nullptr, std::move(inputs)),
387+
RnExecutorchError);
388+
}
389+
390+
TEST_F(GemmaAudioTest, GenerateMultimodalAudioProducesOutput) {
391+
std::vector<float> wav = loadAudio();
392+
ASSERT_FALSE(wav.empty())
393+
<< "test_audio_float.raw missing on device - check run_tests.sh assets";
394+
395+
std::string prompt =
396+
formatGemma(std::string(kGemmaAudioToken) + " Transcribe the audio.");
397+
MultimodalInputs inputs{.audios = AudioInputs{.waveforms = {std::move(wav)},
398+
.token = kGemmaAudioToken}};
399+
std::string output =
400+
model_->generateMultimodal(prompt, nullptr, std::move(inputs));
401+
402+
EXPECT_FALSE(output.empty());
403+
EXPECT_GT(model_->getGeneratedTokenCount(), 0);
404+
}
405+
406+
TEST_F(GemmaAudioTest, GenerateMultimodalInterleavedTextAndAudio) {
407+
std::string prompt = formatGemma("Listen: " + std::string(kGemmaAudioToken) +
408+
" then summarise it.");
409+
MultimodalInputs inputs{.audios = AudioInputs{.waveforms = {loadAudio()},
410+
.token = kGemmaAudioToken}};
411+
std::string output =
412+
model_->generateMultimodal(prompt, nullptr, std::move(inputs));
413+
414+
EXPECT_FALSE(output.empty());
415+
}

‎packages/react-native-executorch/common/rnexecutorch/tests/run_tests.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ MODELS=(
8686
"lfm2_5_vl_quantized_xnnpack_v2.pte|https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/${HF_VERSION_TAG}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte"
8787
"lfm2_vl_tokenizer.json|https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/${HF_VERSION_TAG}/vl_1_6b/tokenizer.json"
8888
"lfm2_vl_tokenizer_config.json|https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/${HF_VERSION_TAG}/vl_1_6b/tokenizer_config.json"
89+
"gemma4_e2b_mm_xnnpack.pte|https://huggingface.co/software-mansion/react-native-executorch-gemma-4-multimodal/${HF_VERSION_TAG}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte"
90+
"gemma_tokenizer.json|https://huggingface.co/software-mansion/react-native-executorch-gemma-4/${HF_VERSION_TAG}/e2b/tokenizer.json"
8991
"yolo26n-seg.pte|https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/${HF_VERSION_TAG}/n/xnnpack/yolo26_seg_n_xnnpack_fp32.pte"
9092
"segmentation_image.jpg|https://upload.wikimedia.org/wikipedia/commons/thumb/8/85/Collage_audi.jpg/1280px-Collage_audi.jpg"
9193
"yolo26n-pose.pte|https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/${HF_VERSION_TAG}/xnnpack/yolo26_pose_n_xnnpack_fp32.pte"
@@ -212,7 +214,7 @@ models_for_test() {
212214
TokenizerModuleTests) echo "tokenizer.json" ;;
213215
SpeechToTextTests) echo "whisper_tiny_en_xnnpack.pte whisper_tokenizer.json fsmn-vad_xnnpack.pte" ;;
214216
TextToSpeechTests) echo "kokoro_duration_predictor.pte kokoro_synthesizer.pte kokoro_af_heart.bin kokoro_us_lexicon.json kokoro_en_tagger.json kokoro_us_phonemizer.pte" ;;
215-
LLMTests) echo "smolLm2_135M_8da4w.pte smollm_tokenizer.json lfm2_5_vl_quantized_xnnpack_v2.pte lfm2_vl_tokenizer.json lfm2_vl_tokenizer_config.json test_image.jpg" ;;
217+
LLMTests) echo "smolLm2_135M_8da4w.pte smollm_tokenizer.json lfm2_5_vl_quantized_xnnpack_v2.pte lfm2_vl_tokenizer.json lfm2_vl_tokenizer_config.json test_image.jpg gemma4_e2b_mm_xnnpack.pte gemma_tokenizer.json" ;;
216218
TextToImageTests) echo "t2i_tokenizer.json t2i_encoder.pte t2i_unet.pte t2i_decoder.pte" ;;
217219
InstanceSegmentationTests) echo "yolo26n-seg.pte segmentation_image.jpg" ;;
218220
PoseEstimationTests) echo "yolo26n-pose.pte" ;;

‎packages/react-native-executorch/common/rnexecutorch/tests/unit/SamplerTest.cpp‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88

99
#include <gtest/gtest.h>
10+
#include <runner/irunner.h>
1011
#include <runner/sampler.h>
1112
#include <vector>
1213

@@ -26,7 +27,7 @@ std::vector<int> sampleMany(Sampler &s, std::vector<T> logits,
2627

2728
// 1. Repetition penalty on positive logit: token 0 should be sampled less.
2829
TEST(SamplerTest, RepetitionPenaltyReducesPositiveLogit) {
29-
Sampler s(2, 1.0f, 1.0f, 0, 0.0f, 1.3f);
30+
Sampler s(2, {.temperature = 1.0f, .topp = 1.0f, .repetition_penalty = 1.3f});
3031
std::vector<float> logits = {1.0f, 1.0f};
3132
std::vector<uint64_t> recent = {0};
3233
auto counts = sampleMany(s, logits, recent, 2000);
@@ -36,7 +37,7 @@ TEST(SamplerTest, RepetitionPenaltyReducesPositiveLogit) {
3637
// 2. Repetition penalty on negative logit: penalised token should appear even
3738
// less.
3839
TEST(SamplerTest, RepetitionPenaltyMultipliesNegativeLogit) {
39-
Sampler s(2, 1.0f, 1.0f, 0, 0.0f, 1.5f);
40+
Sampler s(2, {.temperature = 1.0f, .topp = 1.0f, .repetition_penalty = 1.5f});
4041
std::vector<float> logits = {0.0f, -1.0f};
4142
std::vector<uint64_t> recent = {1};
4243
auto counts = sampleMany(s, logits, recent, 2000);
@@ -45,8 +46,10 @@ TEST(SamplerTest, RepetitionPenaltyMultipliesNegativeLogit) {
4546

4647
// 3. No recent tokens — penalty has no effect.
4748
TEST(SamplerTest, RepetitionPenaltyNoRecentTokensHasNoEffect) {
48-
Sampler baseline(2, 1.0f, 1.0f, 0, 0.0f, 1.0f);
49-
Sampler penalised(2, 1.0f, 1.0f, 0, 0.0f, 2.0f);
49+
Sampler baseline(
50+
2, {.temperature = 1.0f, .topp = 1.0f, .repetition_penalty = 1.0f});
51+
Sampler penalised(
52+
2, {.temperature = 1.0f, .topp = 1.0f, .repetition_penalty = 2.0f});
5053
std::vector<float> logits_b = {1.0f, 1.0f};
5154
std::vector<float> logits_p = {1.0f, 1.0f};
5255
std::vector<uint64_t> recent = {};
@@ -57,7 +60,7 @@ TEST(SamplerTest, RepetitionPenaltyNoRecentTokensHasNoEffect) {
5760

5861
// 4. Min-p truncation: token with very low probability is excluded.
5962
TEST(SamplerTest, MinPFiltersTailTokens) {
60-
Sampler s(3, 1.0f, 1.0f, 0, 0.1f, 1.0f);
63+
Sampler s(3, {.temperature = 1.0f, .topp = 1.0f, .min_p = 0.1f});
6164
std::vector<float> logits = {5.0f, -5.0f, -5.0f};
6265
std::vector<uint64_t> recent = {};
6366
auto counts = sampleMany(s, logits, recent, 1000);
@@ -68,15 +71,15 @@ TEST(SamplerTest, MinPFiltersTailTokens) {
6871

6972
// 5. Min-p = 0 disables filtering.
7073
TEST(SamplerTest, MinPZeroDisablesFiltering) {
71-
Sampler s(3, 0.0f, 1.0f, 0, 0.0f, 1.0f);
74+
Sampler s(3, {.temperature = 0.0f, .topp = 1.0f});
7275
std::vector<float> logits = {1.0f, -1000.0f, -1000.0f};
7376
std::vector<uint64_t> recent = {};
7477
EXPECT_EQ(s.sample(logits.data(), recent), 0);
7578
}
7679

7780
// 6. Min-p + top-p stacked.
7881
TEST(SamplerTest, MinPAndToppStack) {
79-
Sampler s(4, 1.0f, 0.5f, 0, 0.2f, 1.0f);
82+
Sampler s(4, {.temperature = 1.0f, .topp = 0.5f, .min_p = 0.2f});
8083
std::vector<float> logits = {5.0f, 2.0f, -2.0f, -5.0f};
8184
std::vector<uint64_t> recent = {};
8285
auto counts = sampleMany(s, logits, recent, 2000);

0 commit comments

Comments
 (0)