singing_clone.py

"""
Singing model + cloned voice: all emotion tags on one neutral sentence.

Combines the two interesting parts: the ModelsLab/omnivoice-singing
finetune (singing plus emotion tags) and voice cloning from your own
reference recording.

Two deliberate choices in the setup:

  * All emotion cases use the SAME neutral sentence. Its wording carries no
    feeling of its own, so whatever you hear has to come from the tag. A
    sentence like "what wonderful news" would sound happy even untagged and
    would tell you nothing.

  * One cloned voice throughout, so the speaker never changes between
    cases. With auto voice a difference could always be the new speaker
    rather than the tag.

All tags were trained on English data, so German is extrapolation. The
singing section pairs German with English for exactly that reason.

Run from the project root (omnivoice_lab):
    uv run --project OmniVoice python scripts/singing_clone.py
"""

import os
import time

# Force huggingface_hub offline BEFORE anything imports it. Every file we
# need is already on disk, so a network call would mean something is wrong.
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")

import soundfile as sf
import torch
from dotenv import load_dotenv
from omnivoice import OmniVoice

load_dotenv()

MODEL_DIR = os.getenv("OMNIVOICE_SINGING_MODEL_DIR")
AUDIO_DIR = os.path.join("data", "audio_ref")
REFERENCE = os.getenv("OMNIVOICE_REF_NAME", "my_voice_berlin_k")

REF_AUDIO = os.path.join(AUDIO_DIR, f"{REFERENCE}.wav")
REF_TEXT_FILE = os.path.join(AUDIO_DIR, f"reference_txt_{REFERENCE}.txt")

OUTPUT_DIR = os.path.join("data", "output", "singing_clone")

# The model card recommends 3.0 (default is 2.0) to make tags pronounced.
GUIDANCE_SCALE = 3.0

# Neutral on purpose: no word in it leans towards any emotion.
NEUTRAL_DE = "Ich habe heute Morgen deine Nachricht gelesen."

# strong per the model card
STRONG_TAGS = ["happy", "sad", "angry", "nervous", "whisper"]
# weak: only ~190 training clips each, expect them to barely register
WEAK_TAGS = ["calm", "excited"]

SINGING_DE = "Hoch auf dem gelben Wagen sitz ich beim Schwager vorn."
SINGING_EN = "Twinkle twinkle little star, how I wonder what you are."


def build_cases():
    """(name, text, language) triples, grouped by what they are testing."""
    cases = [
        # Control: the neutral sentence untagged. Everything below is judged
        # against this one.
        ("00_control", NEUTRAL_DE, "de"),
    ]

    for index, tag in enumerate(STRONG_TAGS, start=1):
        cases.append((f"{index:02d}_{tag}", f"[{tag}] {NEUTRAL_DE}", "de"))

    for index, tag in enumerate(WEAK_TAGS, start=len(STRONG_TAGS) + 1):
        cases.append((f"{index:02d}_{tag}_weak", f"[{tag}] {NEUTRAL_DE}", "de"))

    cases += [
        # Singing, German against an English control.
        ("10_singing_de", f"[singing] {SINGING_DE}", "de"),
        ("11_singing_en", f"[singing] {SINGING_EN}", "en"),

        # Combined tags, the form shown on the model card.
        ("12_singing_sad_de", f"[singing] [sad] {SINGING_DE}", "de"),
        ("13_singing_happy_de", f"[singing] [happy] {SINGING_DE}", "de"),
        ("14_singing_sad_en", f"[singing] [sad] {SINGING_EN}", "en"),

        # Two tags that pull in different directions -- does one win, do they
        # blend, or does the model give up?
        ("15_whisper_angry", f"[whisper] [angry] {NEUTRAL_DE}", "de"),
    ]
    return cases


def main():
    if not MODEL_DIR or not os.path.isdir(MODEL_DIR):
        raise SystemExit(
            f"Model directory not found. Please set OMNIVOICE_SINGING_MODEL_DIR in your environment or .env (current value: {MODEL_DIR})"
        )

    if not os.path.exists(REF_AUDIO):
        raise SystemExit(
            f"\n[ERROR] Reference audio file missing!\n"
            f"Expected file: {REF_AUDIO}\n\n"
            f"How to fix:\n"
            f"1. Record a 3-10s mono WAV (24 kHz recommended) and place it in '{AUDIO_DIR}'.\n"
            f"2. Set OMNIVOICE_REF_NAME='{REFERENCE}' in .env (or adjust REFERENCE in the script).\n"
            f"See {os.path.join(AUDIO_DIR, 'README.md')} for details."
        )

    # Check for transcript under reference_txt_<name>.txt or <name>.txt
    transcript_path = REF_TEXT_FILE
    if not os.path.exists(transcript_path):
        alt_path = os.path.join(AUDIO_DIR, f"{REFERENCE}.txt")
        if os.path.exists(alt_path):
            transcript_path = alt_path
        else:
            raise SystemExit(
                f"\n[ERROR] Reference transcript file missing!\n"
                f"Expected file: {REF_TEXT_FILE} (or {alt_path})\n\n"
                f"How to fix:\n"
                f"1. Create a text file containing the exact spoken text of '{REFERENCE}.wav'.\n"
                f"2. Save it as '{REF_TEXT_FILE}'.\n"
                f"Providing the transcript is required to prevent unwanted Whisper model downloads."
            )

    with open(transcript_path, encoding="utf-8") as handle:
        ref_text = handle.read().strip()

    os.makedirs(OUTPUT_DIR, exist_ok=True)

    print(f"Loading model from {MODEL_DIR} ...")
    started = time.perf_counter()
    model = OmniVoice.from_pretrained(
        MODEL_DIR,
        device_map="cuda:0",
        dtype=torch.float16,
    )
    print(f"Model loaded in {time.perf_counter() - started:.1f} s")

    prompt = model.create_voice_clone_prompt(ref_audio=REF_AUDIO, ref_text=ref_text)
    print(f"Reference      : {REF_AUDIO}")
    print(f"guidance_scale : {GUIDANCE_SCALE}\n")

    sample_rate = getattr(model, "sampling_rate", None) or 24000

    # Throwaway call so the first real case does not carry the CUDA warm-up.
    model.generate(text="Aufwärmen.", language="de", voice_clone_prompt=prompt)
    if torch.cuda.is_available():
        torch.cuda.synchronize()

    for name, text, language in build_cases():
        started = time.perf_counter()
        audio = model.generate(
            text=text,
            language=language,
            voice_clone_prompt=prompt,
            guidance_scale=GUIDANCE_SCALE,
        )
        if torch.cuda.is_available():
            torch.cuda.synchronize()
        elapsed = time.perf_counter() - started

        wav = audio[0]
        duration = len(wav) / sample_rate
        sf.write(os.path.join(OUTPUT_DIR, f"{name}.wav"), wav, sample_rate)

        print(f"{name:<22} audio {duration:5.2f} s  compute {elapsed:5.2f} s")

    print(f"\nWrote files to {OUTPUT_DIR}")
    print("Listen to 00_control first, then the tagged versions of the same")
    print("sentence. Then 10 vs 11 for German against English singing.")


if __name__ == "__main__":
    main()