Skip to content

Commit e05269a

Browse files
msluszniakmsluszniak
authored
feat(tts): add Chinese to Kokoro (#1557)
## Description - Bumps phonemis to v0.3.2 (Mandarin G2P matching misaki's Kokoro v1.0). - Adds `models.textToSpeech.KOKORO.ZH` (XNNPACK + Core ML) with 7 of the 8 upstream `zf_*`/`zm_*` voices; `zf_xiaoni` is unintelligible in upstream PyTorch Kokoro too. Voices and the zh lexicon/segmenter are on HF `v0.11.0`. - Adds Chinese to the speech app's Kokoro screen. ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [x] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [ ] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [ ] Android ### Testing instructions - `yarn workspace react-native-executorch test` and `scripts/run-native-tests.sh -R Phonemizer`. - Speech app > Kokoro > Chinese, synthesize the sample text. ### Screenshots ### Related issues Closes #1502 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes
1 parent d412e86 commit e05269a

8 files changed

Lines changed: 74 additions & 11 deletions

File tree

‎.cspell-wordlist.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,3 +424,10 @@ denoiser
424424
imageembeddings
425425
unparseable
426426
zipdepth
427+
xiaobei
428+
xiaoxiao
429+
xiaoyi
430+
yunjian
431+
yunxi
432+
yunxia
433+
yunyang

‎apps/benchmarks/scripts/variant-sizes.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,5 +418,14 @@
418418
"https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/coreml/clip_vit_base_patch32_text_coreml_int8.pte": 63921048,
419419
"https://huggingface.co/software-mansion/react-native-executorch-blazeface/resolve/v0.11.0/vulkan/blazeface_vulkan_fp16.pte": 372866,
420420
"https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/tokenizer.json": 711544,
421-
"https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740
421+
"https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740,
422+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/lexicon.json": 2334963,
423+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/segmenter.json": 8288194,
424+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaobei.bin": 522240,
425+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoxiao.bin": 522240,
426+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoyi.bin": 522240,
427+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunjian.bin": 522240,
428+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxi.bin": 522240,
429+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxia.bin": 522240,
430+
"https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunyang.bin": 522240
422431
}

‎apps/speech/app/kokoro-text-to-speech/index.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ const LANGUAGE_OPTIONS = [
2525
{ label: 'Hindi', value: 'HI' as const },
2626
{ label: 'Polish', value: 'PL' as const },
2727
{ label: 'German', value: 'DE' as const },
28+
{ label: 'Chinese', value: 'ZH' as const },
2829
];
2930

3031
type KokoroLanguage = (typeof LANGUAGE_OPTIONS)[number]['value'];
@@ -56,6 +57,7 @@ const SAMPLE_TEXTS: Record<KokoroLanguage, string> = {
5657
HI: 'कोकोरो एक छोटा टेक्स्ट-टू-स्पीच मॉडल है जो पूरी तरह से आपके डिवाइस पर चलता है।',
5758
PL: 'Kokoro to niewielki model syntezy mowy, który działa w całości na twoim urządzeniu, bez połączenia z internetem.',
5859
DE: 'Kokoro ist ein kompaktes Sprachsynthesemodell, das vollständig auf deinem Gerät läuft, ganz ohne Internetverbindung.',
60+
ZH: 'Kokoro 是一个小巧的语音合成模型，完全在你的设备上运行，无需联网。',
5961
};
6062
// cspell:enable
6163

‎docs/docs/02-extensions/speech/02-text-to-speech.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ The library provides ready-to-use Text-to-Speech models from the [Software Mansi
175175
| Model | Variants | Sub-Models & Assets | Sample Rate | Size Range | Supported Backends | Supported Languages | Notes |
176176
| :----------------------------- | :----------------------------------------------------------------------- | :--------------------------------------------------------------------------- | :---------- | :---------- | :------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
177177
| **Supertonic 3** | [See](../../06-api-reference/variables/models.md#texttospeechsupertonic) | Text Encoder, Duration Predictor, Vector Estimator, Vocoder, Voice Styles | 44.1 kHz | 398 MB | XNNPACK (CPU), MLX (Apple), Vulkan (Android) | [English, Spanish, French, German, Korean, Japanese & more](../../06-api-reference/react-native-executorch/namespaces/speech/variables/SUPERTONIC_SUPPORTED_LANGUAGES.md) | Faster generation and broad multilingual coverage across 10 bundled [speaker styles](../../06-api-reference/variables/SUPERTONIC_DEFAULT_VOICE_NAMES.md), with slightly lower voice naturalness than Kokoro. |
178-
| **Kokoro (Language Packages)** | [See](../../06-api-reference/variables/models.md#texttospeechkokoro) | Duration Predictor, Synthesizer, G2P Lexicon / Neural Model, Language Voices | 24.0 kHz | 332 MB each | XNNPACK (CPU), Core ML (Apple) | English (US/GB), Spanish, French, Italian, Portuguese, Hindi, Polish, German | Exceptional voice naturalness and intonation per language package, but heavier compute per synthesized chunk. |
178+
| **Kokoro (Language Packages)** | [See](../../06-api-reference/variables/models.md#texttospeechkokoro) | Duration Predictor, Synthesizer, G2P Lexicon / Neural Model, Language Voices | 24.0 kHz | 332 MB each | XNNPACK (CPU), Core ML (Apple) | English (US/GB), Spanish, French, Italian, Portuguese, Hindi, Polish, German, Chinese | Exceptional voice naturalness and intonation per language package, but heavier compute per synthesized chunk. |
179179

180180
:::note Model-Specific Pipelines
181181
Because Text-to-Speech architectures require distinct multi-model orchestration in TypeScript (coordinating phonemizers, duration predictors, flow-matching loops, and neural vocoders), TTS pipelines are model-specific. To use custom voices or fine-tuned weights, provide your modified `.pte` models or custom voice JSON/BIN files matching the [`SupertonicTtsModel`](../../06-api-reference/type-aliases/SupertonicTtsModel.md) or [`KokoroTtsModel`](../../06-api-reference/type-aliases/KokoroTtsModel.md) specifications.

‎packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ TEST_F(PhonemizerTest, AcceptsEverySupportedLanguage) {
2525
// The language string is passed straight through to phonemis, which picks
2626
// the pipeline from it. A profile it does not know throws (below), so this
2727
// is what catches a name drifting apart from the TS side.
28-
for (const auto *lang : {"en-us", "en-gb", "de", "fr", "es", "it", "pl", "pt", "hi"}) {
28+
for (const auto *lang : {"en-us", "en-gb", "de", "fr", "es", "it", "pl", "pt", "hi", "zh"}) {
2929
EXPECT_TRUE(evalBool(std::string("__rnexecutorch_jsi__.speech.createPhonemizer({ lang: '") +
3030
lang + "' }); return true;"))
3131
<< "rejected language: " << lang;

‎packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,17 @@ declare const phonemizerBrand: unique symbol;
1111
* Union of all supported language codes in the G2P pipeline.
1212
* @category Speech / Types
1313
*/
14-
export type PhonemizerLanguage = 'en-us' | 'en-gb' | 'fr' | 'es' | 'it' | 'pt' | 'de' | 'pl' | 'hi';
14+
export type PhonemizerLanguage =
15+
| 'en-us'
16+
| 'en-gb'
17+
| 'fr'
18+
| 'es'
19+
| 'it'
20+
| 'pt'
21+
| 'de'
22+
| 'pl'
23+
| 'hi'
24+
| 'zh';
1525

1626
/**
1727
* Configuration options and asset paths for initializing a {@link Phonemizer}.
@@ -51,9 +61,10 @@ export type PhonemizedText = {
5161
/** IPA transcription of the whole text, the same as `phonemize(text)` returns. */
5262
readonly phonemes: string;
5363
/**
54-
* The whitespace-separated words of the input text, in order. Each maps onto
55-
* one space-separated phoneme group, except numbers, which map onto as many as
56-
* they were spelled out into. Empty if the words could not be matched.
64+
* The whitespace-separated words of the input text (segmented words for
65+
* Chinese), in order. Each maps onto one space-separated phoneme group,
66+
* except numbers, which map onto as many as they were spelled out into.
67+
* Empty if the words could not be matched.
5768
*/
5869
readonly words: readonly PhonemizedWord[];
5970
};

‎packages/react-native-executorch/src/models.ts‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1848,9 +1848,9 @@ const kokoroModelPaths = (
18481848
synthesizer: `${KOKORO_ROOT}/${backend}/${dir}/synthesizer_${variant}_${backend}_fp32.pte`,
18491849
});
18501850

1851-
const kokoroVoices = <const N extends string>(names: readonly N[]) =>
1851+
const kokoroVoices = <const N extends string>(names: readonly N[], root = KOKORO_ROOT) =>
18521852
names.reduce(
1853-
(acc, name) => ({ ...acc, [name]: `${KOKORO_ROOT}/voices/${name}.bin` }),
1853+
(acc, name) => ({ ...acc, [name]: `${root}/voices/${name}.bin` }),
18541854
{} as Record<N, string>
18551855
);
18561856

@@ -1863,13 +1863,31 @@ const kokoroEnglishPhonemizer = (lang: 'en-us' | 'en-gb') => ({
18631863
neuralModelSource: `${KOKORO_PHONEMIZER_ROOT}/${lang}/phonemizer_${lang.replace('-', '_')}.pte`,
18641864
});
18651865

1866-
const kokoroNeuralPhonemizer = <const L extends Exclude<PhonemizerLanguage, 'en-us' | 'en-gb'>>(
1866+
const kokoroNeuralPhonemizer = <
1867+
const L extends Exclude<PhonemizerLanguage, 'en-us' | 'en-gb' | 'zh'>,
1868+
>(
18671869
lang: L
18681870
) => ({
18691871
lang,
18701872
neuralModelSource: `${KOKORO_PHONEMIZER_ROOT}/${lang}/phonemizer_${lang}.pte`,
18711873
});
18721874

1875+
// Chinese is published from the next tag. Its phonemizer segments the text
1876+
// into words, with the segmenter data in the tagger slot, and reads each word
1877+
// from the lexicon. Upstream's zf_xiaoni is left out: it is unintelligible even
1878+
// in the reference PyTorch model.
1879+
const KOKORO_NEXT_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`;
1880+
const KOKORO_ZH_PHONEMIZER = {
1881+
lang: 'zh' as const,
1882+
taggerSource: `${KOKORO_NEXT_ROOT}/phonemizer/zh/segmenter.json`,
1883+
lexiconSource: `${KOKORO_NEXT_ROOT}/phonemizer/zh/lexicon.json`,
1884+
};
1885+
const KOKORO_ZH_VOICES = kokoroVoices(
1886+
['zf_xiaobei', 'zf_xiaoxiao', 'zf_xiaoyi', 'zm_yunjian', 'zm_yunxi', 'zm_yunxia', 'zm_yunyang'],
1887+
KOKORO_NEXT_ROOT
1888+
);
1889+
type KokoroZhVoice = keyof typeof KOKORO_ZH_VOICES;
1890+
18731891
const KOKORO_EN_US_XNNPACK_FP32: KokoroTtsModel<
18741892
'af_heart' | 'af_river' | 'af_sarah' | 'am_adam' | 'am_michael' | 'am_santa'
18751893
> = {
@@ -1926,6 +1944,12 @@ const KOKORO_DE_XNNPACK_FP32: KokoroTtsModel<'df_anna'> = {
19261944
phonemizer: kokoroNeuralPhonemizer('de'),
19271945
voices: kokoroVoices(['df_anna']),
19281946
};
1947+
const KOKORO_ZH_XNNPACK_FP32: KokoroTtsModel<KokoroZhVoice> = {
1948+
name: 'kokoro',
1949+
modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'),
1950+
phonemizer: KOKORO_ZH_PHONEMIZER,
1951+
voices: KOKORO_ZH_VOICES,
1952+
};
19291953

19301954
// Core ML counterparts: the same weights, phonemizer and voices as the XNNPACK
19311955
// presets, only the `.pte` files differ. Their token axis is fixed at 128, so
@@ -1987,6 +2011,12 @@ const KOKORO_DE_COREML_FP32: KokoroTtsModel<'df_anna'> = {
19872011
phonemizer: kokoroNeuralPhonemizer('de'),
19882012
voices: kokoroVoices(['df_anna']),
19892013
};
2014+
const KOKORO_ZH_COREML_FP32: KokoroTtsModel<KokoroZhVoice> = {
2015+
name: 'kokoro',
2016+
modelPaths: kokoroModelPaths('coreml', 'std', 'standard'),
2017+
phonemizer: KOKORO_ZH_PHONEMIZER,
2018+
voices: KOKORO_ZH_VOICES,
2019+
};
19902020

19912021
// =============================================================================
19922022
// Privacy Filter
@@ -3622,6 +3652,10 @@ export const models = {
36223652
XNNPACK_FP32: KOKORO_DE_XNNPACK_FP32,
36233653
COREML_FP32: KOKORO_DE_COREML_FP32,
36243654
}),
3655+
ZH: variants({
3656+
XNNPACK_FP32: KOKORO_ZH_XNNPACK_FP32,
3657+
COREML_FP32: KOKORO_ZH_COREML_FP32,
3658+
}),
36253659
},
36263660
},
36273661

‎packages/react-native-executorch/third-party/common/phonemis‎

Submodule phonemis updated 28 files

0 commit comments

Comments
 (0)