Merge fix/multispeaker-voice-drift into main

Author: ncoder-ai

.env.example

@@ -43,6 +43,9 @@ FISH_STREAM_CHUNK_TOKENS=32
 # Silence at each text-chunk boundary (ms). Chunks otherwise join with ~0.15 s
 # of pause versus ~0.4 s between sentences inside a chunk. 0 disables.
 FISH_CHUNK_PAUSE_MS=400
+# Multi-speaker: start a new generation at each speaker change (prevents voices
+# drifting into the first speaker's). 0 = old grouping.
+FISH_SPLIT_SPEAKERS=1
 
 # --- Auth ---
 # Require this bearer token on requests (leave blank for an open server).

OPENAI_API_README.md

@@ -196,6 +196,11 @@ accepted via `/v1/tts`.
   its last word, so chunks joined edge to edge sound rushed next to the ~0.4 s
   the model leaves between sentences within a chunk. The engine measures each
   chunk's trailing silence and pads up to this target. `0` disables.
+- `FISH_SPLIT_SPEAKERS` (1) — in multi-speaker input, start a new generation at
+  every speaker change (consecutive lines by one speaker stay together). With
+  several speakers in one generation, the model drifts other speakers toward the
+  voice it opened with; a 3-voice scene had wrong-voice lines in 10/24 renders
+  versus 0/24 with this on, at no speed cost. `0` restores the old grouping.
 - `FISH_RAS` (1) — Repetition Aware Sampling; `0` disables it (diagnostics).
 - `FISH_REF_CACHE` (on) — reuse encoded reference voices across requests. The
   voice add/delete endpoints invalidate it; set `off` if you edit files under

compose.yml

@@ -42,6 +42,7 @@ services:
       FISH_ASR_MODEL: "${FISH_ASR_MODEL:-small}"  # faster-whisper model (small=multilingual)
       FISH_ASR_DEVICE: "${FISH_ASR_DEVICE:-cpu}"  # cpu | cuda:0 (cpu avoids VRAM contention)
       FISH_CHUNK_PAUSE_MS: "${FISH_CHUNK_PAUSE_MS:-400}"  # pause at text-chunk boundaries
+      FISH_SPLIT_SPEAKERS: "${FISH_SPLIT_SPEAKERS:-1}"    # new generation per speaker change
       FISH_BATCH_SIZE: "${FISH_BATCH_SIZE:-1}"    # >1 = continuous batching (needs FISH_QUANTIZE=none)
       FISH_CONCURRENCY: "${FISH_CONCURRENCY:-}"   # empty = FISH_BATCH_SIZE
       FISH_QUEUE_TIMEOUT: "${FISH_QUEUE_TIMEOUT:-300}"

fish_speech/models/text2semantic/inference.py

@@ -50,6 +50,12 @@ def multinomial_sample_one_no_sync(probs_sort):
 # FISH_RAS=0 disables Repetition Aware Sampling (resampling a token that
 # repeats within the window at a higher temperature).
 RAS_ENABLED = os.environ.get("FISH_RAS", "1") != "0"
+# Start a new text batch at every speaker change so each generation opens with
+# its own speaker's voice. With several speakers in one batch the model drifts
+# other speakers toward the batch's opening voice (measured: 10/24 renders of a
+# 3-voice scene had wrong-voice lines vs 0/24 when split). FISH_SPLIT_SPEAKERS=0
+# restores the old grouping.
+SPLIT_ON_SPEAKER_CHANGE = os.environ.get("FISH_SPLIT_SPEAKERS", "1") != "0"
 RAS_HIGH_TEMP = 1.0
 RAS_HIGH_TOP_P = 0.9
 
@@ -751,7 +757,10 @@ def split_text_by_speaker(text: str) -> list[str]:
 
 
 def group_turns_into_batches(
-    turns: list[str], max_speakers: int = 3, max_bytes: int = 300
+    turns: list[str],
+    max_speakers: int = 3,
+    max_bytes: int = 300,
+    split_on_speaker_change: bool = False,
 ) -> list[str]:
     """
     Group turns into batches based on speaker count or byte limit.
@@ -768,10 +777,22 @@ def group_turns_into_batches(
     current_batch = []
     current_bytes = 0
 
+    speaker_re = re.compile(r"<\|speaker:(\d+)\|>")
+    current_speaker = None
+
     for turn in turns:
         turn_bytes = len(turn.encode("utf-8"))
+        m = speaker_re.match(turn)
+        speaker = m.group(1) if m else None
+        # Each generation then starts from its own speaker tag; within one
+        # generation the model tends to carry the opening voice into later
+        # turns (other speakers drift toward speaker 0's voice).
+        speaker_changed = (
+            split_on_speaker_change and current_batch and speaker != current_speaker
+        )
+        current_speaker = speaker
 
-        would_exceed_speakers = len(current_batch) >= max_speakers
+        would_exceed_speakers = len(current_batch) >= max_speakers or speaker_changed
         would_exceed_bytes = current_bytes + turn_bytes > max_bytes and current_batch
 
         if would_exceed_speakers or would_exceed_bytes:
@@ -941,7 +962,10 @@ def generate_long_steps(
     turns = split_text_by_speaker(text)
     if turns:
         batches = group_turns_into_batches(
-            turns, max_speakers=5, max_bytes=chunk_length
+            turns,
+            max_speakers=5,
+            max_bytes=chunk_length,
+            split_on_speaker_change=SPLIT_ON_SPEAKER_CHANGE,
         )
     else:
         batches = [text]