|
3 | 3 | Expects, already placed on the VM before this script runs: |
4 | 4 | /content/ref.wav -- reference voice clip (via `colab upload`) |
5 | 5 | /content/batch.jsonl -- indextts2 batch tasks (via `colab upload`) |
| 6 | + /content/hf_token -- optional HuggingFace token (via `colab upload`), |
| 7 | + improves model-download stability/speed over |
| 8 | + anonymous requests. Read once, then deleted. |
6 | 9 |
|
7 | 10 | Produces: |
8 | 11 | /content/output.wav -- concatenated synthesis result |
| 12 | +
|
| 13 | +Each step gets its own timeout enforced by *this* script (not just the |
| 14 | +outer `colab exec --timeout`), and prints a heartbeat line periodically |
| 15 | +so a stuck step is diagnosable instead of going silent for the whole |
| 16 | +outer deadline. A run on 2026-08-16 spent 28 minutes with zero log |
| 17 | +output before hitting the outer timeout during model download -- |
| 18 | +these heartbeats plus per-step timeouts are the direct fix for that. |
9 | 19 | """ |
| 20 | +import os |
10 | 21 | import subprocess |
11 | 22 | import sys |
| 23 | +import time |
12 | 24 | from pathlib import Path |
13 | 25 |
|
14 | 26 | WORK = Path("/content") |
|
17 | 29 | BATCH_FILE = WORK / "batch.jsonl" |
18 | 30 | VOICE_FILE = WORK / "ref.wav" |
19 | 31 | OUTPUT_FILE = WORK / "output.wav" |
| 32 | +HF_TOKEN_FILE = WORK / "hf_token" |
| 33 | +STEP_LOG = WORK / "_step.log" |
| 34 | + |
| 35 | +HEARTBEAT_SECONDS = 60 |
| 36 | +STALL_HEARTBEATS_BEFORE_WARNING = 5 # 5 min of zero new output -> call it out |
| 37 | + |
| 38 | + |
| 39 | +def run(cmd, cwd=None, timeout=600): |
| 40 | + print(f"\n>>> {' '.join(cmd)} (timeout={timeout}s)", flush=True) |
| 41 | + start = time.monotonic() |
| 42 | + stall_count = 0 |
| 43 | + last_size = -1 |
| 44 | + with open(STEP_LOG, "wb") as logf: |
| 45 | + proc = subprocess.Popen(cmd, cwd=cwd, stdout=logf, stderr=subprocess.STDOUT) |
| 46 | + while proc.poll() is None: |
| 47 | + elapsed = time.monotonic() - start |
| 48 | + if elapsed > timeout: |
| 49 | + proc.kill() |
| 50 | + proc.wait() |
| 51 | + _print_tail() |
| 52 | + raise RuntimeError( |
| 53 | + f"command exceeded {timeout}s timeout after {elapsed:.0f}s: {' '.join(cmd)}" |
| 54 | + ) |
| 55 | + time.sleep(HEARTBEAT_SECONDS) |
| 56 | + size = STEP_LOG.stat().st_size |
| 57 | + grew = size != last_size |
| 58 | + stall_count = 0 if grew else stall_count + 1 |
| 59 | + note = "" if grew else f" [no new output for {stall_count * HEARTBEAT_SECONDS}s]" |
| 60 | + print(f" ... still running ({elapsed:.0f}s elapsed, {size} bytes so far){note}", flush=True) |
| 61 | + if stall_count >= STALL_HEARTBEATS_BEFORE_WARNING: |
| 62 | + print(f" ⚠ possible stall: no output growth for " |
| 63 | + f"{stall_count * HEARTBEAT_SECONDS}s (will still respect the {timeout}s timeout)", flush=True) |
| 64 | + last_size = size |
| 65 | + |
| 66 | + _print_tail() |
| 67 | + if proc.returncode != 0: |
| 68 | + raise RuntimeError(f"command failed ({proc.returncode}): {' '.join(cmd)}") |
| 69 | + |
20 | 70 |
|
| 71 | +def _print_tail(max_bytes=20000): |
| 72 | + data = STEP_LOG.read_bytes() |
| 73 | + if len(data) > max_bytes: |
| 74 | + print(f"[... truncated, showing last {max_bytes} bytes of {len(data)} ...]") |
| 75 | + data = data[-max_bytes:] |
| 76 | + sys.stdout.write(data.decode(errors="replace")) |
| 77 | + sys.stdout.flush() |
21 | 78 |
|
22 | | -def run(cmd, cwd=None): |
23 | | - print(f"\n>>> {' '.join(cmd)}", flush=True) |
24 | | - result = subprocess.run(cmd, cwd=cwd) |
25 | | - if result.returncode != 0: |
26 | | - raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") |
| 79 | + |
| 80 | +def setup_hf_token(): |
| 81 | + if not HF_TOKEN_FILE.is_file(): |
| 82 | + print(">> no /content/hf_token uploaded; HF downloads will be unauthenticated " |
| 83 | + "(slower/rate-limited on the free tier).") |
| 84 | + return |
| 85 | + token = HF_TOKEN_FILE.read_text().strip() |
| 86 | + HF_TOKEN_FILE.unlink() # don't leave it sitting on the VM disk |
| 87 | + if token: |
| 88 | + os.environ["HF_TOKEN"] = token |
| 89 | + print(">> HF_TOKEN set from uploaded credential.") |
| 90 | + |
| 91 | + |
| 92 | +def count_batch_tasks(): |
| 93 | + with open(BATCH_FILE, encoding="utf-8") as f: |
| 94 | + return sum(1 for line in f if line.strip()) |
27 | 95 |
|
28 | 96 |
|
29 | 97 | def main(): |
30 | 98 | assert BATCH_FILE.is_file(), f"missing {BATCH_FILE}, did the upload step run?" |
31 | 99 | assert VOICE_FILE.is_file(), f"missing {VOICE_FILE}, did the upload step run?" |
32 | 100 |
|
| 101 | + setup_hf_token() |
| 102 | + |
33 | 103 | if not REPO.is_dir(): |
34 | 104 | run(["git", "clone", "--depth", "1", |
35 | | - "https://github.com/index-tts/index-tts.git", str(REPO)]) |
| 105 | + "https://github.com/index-tts/index-tts.git", str(REPO)], timeout=180) |
36 | 106 |
|
37 | | - run(["uv", "sync"], cwd=str(REPO)) |
| 107 | + run(["uv", "sync"], cwd=str(REPO), timeout=600) |
38 | 108 | run(["uv", "run", "indextts2", "download", "--model-dir", str(MODEL_DIR)], |
39 | | - cwd=str(REPO)) |
| 109 | + cwd=str(REPO), timeout=1200) |
40 | 110 | run(["uv", "run", "indextts2", "check", "--model-dir", str(MODEL_DIR), |
41 | | - "--device", "cuda"], cwd=str(REPO)) |
| 111 | + "--device", "cuda"], cwd=str(REPO), timeout=90) |
| 112 | + |
| 113 | + num_tasks = count_batch_tasks() |
| 114 | + batch_timeout = 120 + 90 * num_tasks |
42 | 115 | run(["uv", "run", "indextts2", "batch", |
43 | 116 | "--batch-file", str(BATCH_FILE), |
44 | 117 | "--model-dir", str(MODEL_DIR), |
45 | 118 | "--concat", "--output", str(OUTPUT_FILE), |
46 | 119 | "--no-cuda-kernel", "--force", "--verbose"], |
47 | | - cwd=str(REPO)) |
| 120 | + cwd=str(REPO), timeout=batch_timeout) |
48 | 121 |
|
49 | 122 | assert OUTPUT_FILE.is_file(), "synthesis finished but output.wav was not created" |
50 | 123 | size = OUTPUT_FILE.stat().st_size |
|
0 commit comments