Skip to content

Commit fb210c3

Browse files
NorbertKlockiewiczclaude
NorbertKlockiewicz
and
claude
authored
feat: add MLX builds for LFM2.5 and privacy-filter models (#1266)
## Description Adds MLX (Apple GPU) builds for the LFM2.5 family and the privacy-filter models, wiring the HF-hosted MLX variants into the model registry so they default to MLX on iOS (with the existing XNNPACK builds as the simulator / Android fallback): - LFM2.5 text — 350M, 1.2B - LFM2.5-VL — 450M, 1.6B - Privacy filter — openai, nemotron Runner changes needed to drive the new builds: - `vision_encoder` now reads its declared input dtype from method metadata and converts the preprocessed fp32 pixels to match (fp32 passthrough, bf16, or fp16) instead of hardcoding `Float`. Required for bf16 MLX vision encoders; the fp32 path stays zero-copy. - The multimodal prefiller image-embed splice now handles `fp32 <-> bf16` vision/text-embed dtype pairs (a hybrid where the vision encoder is fp32 and the decoder embeds are bf16), in addition to the existing fp32<->fp16 cases. - New `convert_from_float` helper in `runner/util.h`. Review order: `modelUrls.ts` + `modelRegistry.ts` (the wiring) first, then the three `common/runner` files (the dtype handling). ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [x] New feature (change which adds functionality) ### Tested on - [x] iOS ### Testing instructions 1. On a physical iOS device, load each model via its accessor with the default backend (resolves to MLX on iOS), e.g. `models.llm.lfm2_5_vl_450m()`, `models.privacy_filter.openai()`. 2. Verify the privacy filters classify PII and the LFM2.5 / LFM2.5-VL models generate correctly (the VL models with an image). 3. MLX requires a physical device; on the simulator the accessors fall back to XNNPACK (or pass `{ backend: 'xnnpack' }` explicitly). ### Checklist - [x] I have performed a self-review of my code --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent b180046 commit fb210c3

6 files changed

Lines changed: 158 additions & 25 deletions

File tree

‎packages/react-native-executorch/common/runner/encoders/vision_encoder.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <rnexecutorch/Error.h>
55
#include <rnexecutorch/data_processing/ImageProcessing.h>
66
#include <runner/constants.h>
7+
#include <runner/util.h>
78

89
#include <executorch/extension/tensor/tensor.h>
910
#include <opencv2/opencv.hpp>
@@ -70,6 +71,7 @@ Result<VisionEncoder::ImageShape> VisionEncoder::getInputShape() const {
7071
.height = static_cast<int32_t>(dims[offset + 1]),
7172
.width = static_cast<int32_t>(dims[offset + 2]),
7273
.with_batch = with_batch,
74+
.dtype = input_meta.scalar_type(),
7375
};
7476
}
7577

@@ -124,8 +126,12 @@ Result<EValue> VisionEncoder::encode(const MultimodalInput &input) {
124126
sizes.insert(sizes.begin(), 1);
125127
}
126128

129+
// Preprocessing produces fp32 pixels; convert to the method's declared
130+
// input dtype (`shape.dtype`, already read in getInputShape). Float is a
131+
// passthrough, so the common path stays copy-free.
127132
auto image_tensor = ::executorch::extension::from_blob(
128133
chw.data(), sizes, ::executorch::aten::ScalarType::Float);
134+
image_tensor = ET_UNWRAP(convert_from_float(image_tensor, shape.dtype));
129135

130136
auto result = ET_UNWRAP(module_->execute(kVisionEncoderMethod, image_tensor));
131137
auto out_tensor = result[0].toTensor();

‎packages/react-native-executorch/common/runner/encoders/vision_encoder.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class VisionEncoder : public IEncoder {
2727
struct ImageShape {
2828
int32_t channels, height, width;
2929
bool with_batch;
30+
::executorch::aten::ScalarType dtype;
3031
};
3132

3233
// The method's output EValue aliases the runtime's reusable output buffer,

‎packages/react-native-executorch/common/runner/multimodal_prefiller.cpp‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,20 @@ using ::executorch::runtime::Error;
2424
using ::executorch::runtime::EValue;
2525
using ::executorch::runtime::Result;
2626

27+
namespace {
28+
// Element-wise convert `count` values from `src` (Src) into the raw byte
29+
// buffer `dst` (interpreted as Dst). Used to splice an image-embed tensor of
30+
// one dtype into the fused-embeds buffer of another.
31+
template <typename Src, typename Dst>
32+
void castCopy(const void *src, uint8_t *dst, size_t count) {
33+
const auto *s = static_cast<const Src *>(src);
34+
auto *d = reinterpret_cast<Dst *>(dst);
35+
for (size_t i = 0; i < count; ++i) {
36+
d[i] = static_cast<Dst>(s[i]);
37+
}
38+
}
39+
} // namespace
40+
2741
MultimodalPrefiller::MultimodalPrefiller(
2842
Module &module, MultimodalDecoderRunner &decoder_runner,
2943
tokenizers::HFTokenizer &tokenizer,
@@ -186,24 +200,23 @@ bool MultimodalPrefiller::get_enable_dynamic_shape() const {
186200
uint8_t *dst = embeds_buf.data() + static_cast<size_t>(slot.slot_start) *
187201
static_cast<size_t>(hidden) *
188202
embeds_elem_size;
203+
using ::executorch::aten::ScalarType;
204+
const void *src = vision_tensor.const_data_ptr();
189205
if (vision_dtype == embeds_dtype) {
190-
const uint8_t *src =
191-
static_cast<const uint8_t *>(vision_tensor.const_data_ptr());
192206
std::memcpy(dst, src, visual_elems * embeds_elem_size);
193-
} else if (vision_dtype == ::executorch::aten::ScalarType::Float &&
194-
embeds_dtype == ::executorch::aten::ScalarType::Half) {
195-
const float *src = vision_tensor.const_data_ptr<float>();
196-
auto *dst_h = reinterpret_cast<::executorch::aten::Half *>(dst);
197-
for (size_t i = 0; i < visual_elems; ++i) {
198-
dst_h[i] = ::executorch::aten::Half(src[i]);
199-
}
200-
} else if (vision_dtype == ::executorch::aten::ScalarType::Half &&
201-
embeds_dtype == ::executorch::aten::ScalarType::Float) {
202-
const auto *src = vision_tensor.const_data_ptr<::executorch::aten::Half>();
203-
auto *dst_f = reinterpret_cast<float *>(dst);
204-
for (size_t i = 0; i < visual_elems; ++i) {
205-
dst_f[i] = static_cast<float>(src[i]);
206-
}
207+
} else if (vision_dtype == ScalarType::Float &&
208+
embeds_dtype == ScalarType::Half) {
209+
castCopy<float, ::executorch::aten::Half>(src, dst, visual_elems);
210+
} else if (vision_dtype == ScalarType::Half &&
211+
embeds_dtype == ScalarType::Float) {
212+
castCopy<::executorch::aten::Half, float>(src, dst, visual_elems);
213+
} else if (vision_dtype == ScalarType::Float &&
214+
embeds_dtype == ScalarType::BFloat16) {
215+
// Hybrid VLM: fp32 vision encoder (e.g. XNNPACK) + bf16 decoder embeds.
216+
castCopy<float, ::executorch::aten::BFloat16>(src, dst, visual_elems);
217+
} else if (vision_dtype == ScalarType::BFloat16 &&
218+
embeds_dtype == ScalarType::Float) {
219+
castCopy<::executorch::aten::BFloat16, float>(src, dst, visual_elems);
207220
} else {
208221
ET_CHECK_OR_RETURN_ERROR(
209222
false, InvalidState,

‎packages/react-native-executorch/common/runner/util.h‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,54 @@ convert_to_bfloat16(const ::executorch::extension::TensorPtr &src_tensor) {
170170
return bf16_tensor;
171171
}
172172

173+
/**
174+
* Helper function to convert a float tensor to float16 (Half).
175+
* Creates a new tensor with Half dtype and copies/converts the data.
176+
*/
177+
inline ::executorch::runtime::Result<::executorch::extension::TensorPtr>
178+
convert_to_float16(const ::executorch::extension::TensorPtr &src_tensor) {
179+
ET_CHECK_OR_RETURN_ERROR(
180+
src_tensor->scalar_type() == ::executorch::aten::ScalarType::Float,
181+
InvalidArgument,
182+
"Float16 conversion only supported from Float source data");
183+
184+
const auto num_elements = static_cast<size_t>(src_tensor->numel());
185+
const float *float_data = src_tensor->const_data_ptr<float>();
186+
187+
auto half_tensor = ::executorch::extension::empty_like(
188+
src_tensor, ::executorch::aten::ScalarType::Half);
189+
auto *half_data = half_tensor->mutable_data_ptr<::executorch::aten::Half>();
190+
for (size_t i = 0; i < num_elements; ++i) {
191+
half_data[i] = ::executorch::aten::Half(float_data[i]);
192+
}
193+
194+
return half_tensor;
195+
}
196+
197+
/**
198+
* Convert a Float tensor to `dtype` (Float passthrough, BFloat16, or Half).
199+
* Used to match an exported method's declared input dtype when preprocessing
200+
* produces fp32 data. Returns InvalidArgument for unsupported targets.
201+
*/
202+
inline ::executorch::runtime::Result<::executorch::extension::TensorPtr>
203+
convert_from_float(const ::executorch::extension::TensorPtr &src_tensor,
204+
::executorch::aten::ScalarType dtype) {
205+
using ::executorch::aten::ScalarType;
206+
switch (dtype) {
207+
case ScalarType::Float:
208+
return src_tensor;
209+
case ScalarType::BFloat16:
210+
return convert_to_bfloat16(src_tensor);
211+
case ScalarType::Half:
212+
return convert_to_float16(src_tensor);
213+
default:
214+
ET_CHECK_OR_RETURN_ERROR(
215+
false, InvalidArgument,
216+
"Unsupported target dtype %hhd for float conversion",
217+
static_cast<int8_t>(dtype));
218+
}
219+
}
220+
173221
} // namespace llm
174222
} // namespace extension
175223
} // namespace executorch

‎packages/react-native-executorch/src/constants/modelRegistry.ts‎

Lines changed: 66 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,64 @@ const GEMMA4_E2B_MM_VARIANTS = {
260260
},
261261
};
262262

263+
const LFM2_5_350M_VARIANTS = {
264+
mlx: { base: { ...M.LFM2_5_350M, modelSource: M.LFM2_5_350M_MLX_MODEL } },
265+
xnnpack: { base: M.LFM2_5_350M, quant: M.LFM2_5_350M_QUANTIZED },
266+
};
267+
268+
const LFM2_5_1_2B_INSTRUCT_VARIANTS = {
269+
mlx: {
270+
base: {
271+
...M.LFM2_5_1_2B_INSTRUCT,
272+
modelSource: M.LFM2_5_1_2B_INSTRUCT_MLX_MODEL,
273+
},
274+
},
275+
xnnpack: {
276+
base: M.LFM2_5_1_2B_INSTRUCT,
277+
quant: M.LFM2_5_1_2B_INSTRUCT_QUANTIZED,
278+
},
279+
};
280+
281+
const LFM2_5_VL_1_6B_VARIANTS = {
282+
mlx: {
283+
base: {
284+
...M.LFM2_5_VL_1_6B_QUANTIZED,
285+
modelSource: M.LFM2_5_VL_1_6B_MLX_MODEL,
286+
},
287+
},
288+
xnnpack: { base: M.LFM2_5_VL_1_6B_QUANTIZED },
289+
};
290+
291+
const LFM2_5_VL_450M_VARIANTS = {
292+
mlx: {
293+
base: {
294+
...M.LFM2_5_VL_450M_QUANTIZED,
295+
modelSource: M.LFM2_5_VL_450M_MLX_MODEL,
296+
},
297+
},
298+
xnnpack: { base: M.LFM2_5_VL_450M_QUANTIZED },
299+
};
300+
301+
const PRIVACY_FILTER_OPENAI_VARIANTS = {
302+
mlx: {
303+
base: {
304+
...M.PRIVACY_FILTER_OPENAI,
305+
modelSource: M.PRIVACY_FILTER_OPENAI_MLX_MODEL,
306+
},
307+
},
308+
xnnpack: { base: M.PRIVACY_FILTER_OPENAI },
309+
};
310+
311+
const PRIVACY_FILTER_NEMOTRON_VARIANTS = {
312+
mlx: {
313+
base: {
314+
...M.PRIVACY_FILTER_NEMOTRON,
315+
modelSource: M.PRIVACY_FILTER_NEMOTRON_MLX_MODEL,
316+
},
317+
},
318+
xnnpack: { base: M.PRIVACY_FILTER_NEMOTRON },
319+
};
320+
263321
const EFFICIENTNET_V2_S_VARIANTS = {
264322
xnnpack: {
265323
base: {
@@ -594,20 +652,19 @@ export const models = {
594652
smollm2_1_360m: pair(M.SMOLLM2_1_360M, M.SMOLLM2_1_360M_QUANTIZED),
595653
smollm2_1_1_7b: pair(M.SMOLLM2_1_1_7B, M.SMOLLM2_1_1_7B_QUANTIZED),
596654
phi_4_mini_4b: pair(M.PHI_4_MINI_4B, M.PHI_4_MINI_4B_QUANTIZED),
597-
lfm2_5_350m: pair(M.LFM2_5_350M, M.LFM2_5_350M_QUANTIZED),
598-
lfm2_5_1_2b_instruct: pair(
599-
M.LFM2_5_1_2B_INSTRUCT,
600-
M.LFM2_5_1_2B_INSTRUCT_QUANTIZED
601-
),
655+
lfm2_5_350m: variant(LFM2_5_350M_VARIANTS, { ios: 'mlx' }),
656+
lfm2_5_1_2b_instruct: variant(LFM2_5_1_2B_INSTRUCT_VARIANTS, {
657+
ios: 'mlx',
658+
}),
602659
bielik_v3_0_1_5b: pair(M.BIELIK_V3_0_1_5B, M.BIELIK_V3_0_1_5B_QUANTIZED),
603660
gemma4_e2b: variant(GEMMA4_E2B_VARIANTS, {
604661
ios: 'mlx',
605662
android: 'vulkan',
606663
}),
607664
// Multimodal LLMs — same hook/module as plain LLMs, listed here so users
608665
// pick a model by capability ("LLM") rather than by modality.
609-
lfm2_5_vl_1_6b: base(M.LFM2_5_VL_1_6B_QUANTIZED),
610-
lfm2_5_vl_450m: base(M.LFM2_5_VL_450M_QUANTIZED),
666+
lfm2_5_vl_1_6b: variant(LFM2_5_VL_1_6B_VARIANTS, { ios: 'mlx' }),
667+
lfm2_5_vl_450m: variant(LFM2_5_VL_450M_VARIANTS, { ios: 'mlx' }),
611668
gemma4_e2b_multimodal: variant(GEMMA4_E2B_MM_VARIANTS, {
612669
ios: 'mlx',
613670
android: 'vulkan',
@@ -617,8 +674,8 @@ export const models = {
617674
efficientnet_v2_s: variant(EFFICIENTNET_V2_S_VARIANTS),
618675
},
619676
privacy_filter: {
620-
openai: base(M.PRIVACY_FILTER_OPENAI),
621-
nemotron: base(M.PRIVACY_FILTER_NEMOTRON),
677+
openai: variant(PRIVACY_FILTER_OPENAI_VARIANTS, { ios: 'mlx' }),
678+
nemotron: variant(PRIVACY_FILTER_NEMOTRON_VARIANTS, { ios: 'mlx' }),
622679
},
623680
object_detection: {
624681
ssdlite_320_mobilenet_v3_large: variant(

‎packages/react-native-executorch/src/constants/modelUrls.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,7 @@ export const PHI_4_MINI_4B_QUANTIZED = {
450450
// LFM2.5-1.2B-Instruct
451451
const LFM2_5_1_2B_INSTRUCT_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_fp16.pte`;
452452
const LFM2_5_1_2B_INSTRUCT_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_8da4w.pte`;
453+
export const LFM2_5_1_2B_INSTRUCT_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/mlx/lfm_2_5_1_2b_mlx_int4.pte`;
453454
const LFM2_5_1_2B_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/tokenizer.json`;
454455
const LFM2_5_1_2B_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/tokenizer_config.json`;
455456

@@ -476,6 +477,7 @@ export const LFM2_5_1_2B_INSTRUCT_QUANTIZED = {
476477
// LFM2.5-350M
477478
const LFM2_5_350M_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/xnnpack/lfm_2_5_350m_xnnpack_fp16.pte`;
478479
const LFM2_5_350M_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte`;
480+
export const LFM2_5_350M_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/mlx/lfm_2_5_350m_mlx_int4.pte`;
479481
const LFM2_5_350M_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/tokenizer.json`;
480482
const LFM2_5_350M_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/tokenizer_config.json`;
481483

@@ -527,11 +529,13 @@ export const BIELIK_V3_0_1_5B_QUANTIZED = {
527529

528530
// LFM2.5-VL-1.6B
529531
const LFM2_VL_1_6B_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`;
532+
export const LFM2_5_VL_1_6B_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte`;
530533
const LFM2_VL_1_6B_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/tokenizer.json`;
531534
const LFM2_VL_1_6B_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/tokenizer_config.json`;
532535

533536
// LFM2.5-VL-450M
534537
const LFM2_VL_450M_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`;
538+
export const LFM2_5_VL_450M_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`;
535539
const LFM2_VL_450M_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/tokenizer.json`;
536540
const LFM2_VL_450M_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/tokenizer_config.json`;
537541

@@ -1281,6 +1285,8 @@ export const PRIVACY_FILTER_OPENAI = {
12811285
tokenizerSource: `${URL_PREFIX}-privacy-filter-openai/${PREVIOUS_VERSION_TAG}/tokenizer.json`,
12821286
} as const;
12831287

1288+
export const PRIVACY_FILTER_OPENAI_MLX_MODEL = `${URL_PREFIX}-privacy-filter-openai/${PREVIOUS_VERSION_TAG}/mlx/privacy_filter_openai_mlx_int4.pte`;
1289+
12841290
/**
12851291
* OpenMed/privacy-filter-nemotron — extended PII detector with 55 entity
12861292
* types (adds medical, financial, identity, technical, demographic, etc.).
@@ -1293,6 +1299,8 @@ export const PRIVACY_FILTER_NEMOTRON = {
12931299
tokenizerSource: `${URL_PREFIX}-privacy-filter-nemotron/${PREVIOUS_VERSION_TAG}/tokenizer.json`,
12941300
} as const;
12951301

1302+
export const PRIVACY_FILTER_NEMOTRON_MLX_MODEL = `${URL_PREFIX}-privacy-filter-nemotron/${PREVIOUS_VERSION_TAG}/mlx/privacy_filter_nemotron_mlx_int8.pte`;
1303+
12961304
// Image generation
12971305

12981306
/**

0 commit comments

Comments
 (0)