synthesize #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: synthesize | |
| on: | |
| issues: | |
| types: [opened] | |
| workflow_dispatch: | |
| inputs: | |
| text: | |
| description: "要朗讀的文字（每行一段）" | |
| required: true | |
| default: | | |
| 君不見，黃河之水天上來，奔流到海不復回。 | |
| 人生得意須盡歡，莫使金樽空對月。 | |
| emotion: | |
| description: "情緒" | |
| required: true | |
| type: choice | |
| default: "自然 (neutral)" | |
| options: | |
| - "自然 (neutral)" | |
| - "高興 (happy)" | |
| - "憤怒 (angry)" | |
| - "悲傷 (sad)" | |
| - "低落 (melancholic)" | |
| - "驚訝 (surprised)" | |
| weight: | |
| description: "情緒強度 (0.1 - 1.0)" | |
| required: false | |
| default: "0.85" | |
| model_version: | |
| description: "模型版本" | |
| required: true | |
| type: choice | |
| default: "2.0（貼近參考音檔原始腔調，推薦）" | |
| options: | |
| - "2.0（貼近參考音檔原始腔調，推薦）" | |
| - "2.5（較快，但中文偏大陸腔）" | |
| # Colab's free tier allows only one GPU runtime at a time on this account, | |
| # so serialize all synthesis runs instead of letting them race each other. | |
| # Without `queue: max`, a concurrency group only holds one running + one | |
| # pending run -- a third request silently cancels the second one (no | |
| # failure, so nothing downstream ever notices) before it ever starts. | |
| # `queue: max` makes it a real FIFO queue (up to 100 waiting), so a | |
| # request only gets dropped in the practically-impossible case of >100 | |
| # requests queued at once. | |
| concurrency: | |
| group: colab-gpu-synthesis | |
| cancel-in-progress: false | |
| queue: max | |
| permissions: | |
| contents: write | |
| issues: write | |
| jobs: | |
| synthesize: | |
| if: > | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.issue.labels.*.name, 'synthesize') | |
| runs-on: ubuntu-latest | |
| # A long request can span many chunks (see scripts/parse_issue.py); | |
| # each colab-exec call is individually bounded, but the sum across a | |
| # worst-case chunk count needs headroom. See README for the math. | |
| # 2026-08-16 real test: 5645 chars / 9 chunks estimated ~131 min of | |
| # synthesis alone (9*131s fixed + 5645*1.18s), pushing total realistic | |
| # time close to the previous 150min ceiling with no margin left. | |
| timeout-minutes: 240 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Build issue-form-shaped body (workflow_dispatch path) | |
| if: github.event_name == 'workflow_dispatch' | |
| env: | |
| IN_TEXT: ${{ inputs.text }} | |
| IN_EMOTION: ${{ inputs.emotion }} | |
| IN_WEIGHT: ${{ inputs.weight }} | |
| IN_MODEL_VERSION: ${{ inputs.model_version }} | |
| run: | | |
| { | |
| echo "### 要朗讀的文字" | |
| echo | |
| echo "$IN_TEXT" | |
| echo | |
| echo "### 情緒" | |
| echo | |
| echo "$IN_EMOTION" | |
| echo | |
| echo "### 情緒強度 (0.1 - 1.0)" | |
| echo | |
| echo "$IN_WEIGHT" | |
| echo | |
| echo "### 模型版本" | |
| echo | |
| echo "$IN_MODEL_VERSION" | |
| } > synthetic_body.md | |
| echo "SYNTHETIC_BODY_FILE=synthetic_body.md" >> "$GITHUB_ENV" | |
| - name: Parse request into batch.jsonl | |
| env: | |
| # Never splice untrusted issue content into a shell `run:` block directly — | |
| # passing it through `env:` keeps it out of shell interpolation. | |
| ISSUE_BODY_FROM_ISSUE: ${{ github.event.issue.body }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| ISSUE_BODY="$(cat "$SYNTHETIC_BODY_FILE")" | |
| else | |
| ISSUE_BODY="$ISSUE_BODY_FROM_ISSUE" | |
| fi | |
| export ISSUE_BODY | |
| python3 scripts/parse_issue.py | |
| - name: Install Colab CLI | |
| run: | | |
| # Pin jupyter-kernel-client below 1.0: the 1.0.0 release (2026-08-08) | |
| # renamed KernelClient -> JupyterKernelClient, which breaks | |
| # google-colab-cli 0.6.0's internal `jupyter_kernel_client.KernelClient(...)` | |
| # call. google-colab-cli itself doesn't pin this transitive dependency. | |
| pip install --user 'jupyter-kernel-client<1.0' google-colab-cli | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Write Colab ADC credentials | |
| env: | |
| COLAB_ADC_JSON: ${{ secrets.COLAB_ADC_CREDENTIALS }} | |
| run: | | |
| if [ -z "$COLAB_ADC_JSON" ]; then | |
| echo "::error::COLAB_ADC_CREDENTIALS secret is not set. See README for setup." | |
| exit 1 | |
| fi | |
| mkdir -p "$HOME/.config/gcloud" | |
| printf '%s' "$COLAB_ADC_JSON" > "$HOME/.config/gcloud/application_default_credentials.json" | |
| - name: Provision Colab T4 session | |
| id: colab_new | |
| run: | | |
| SESSION="ci-${{ github.run_id }}" | |
| echo "session=$SESSION" >> "$GITHUB_OUTPUT" | |
| # Free-tier accounts allow only one GPU runtime at a time, so a | |
| # stray interactive Colab tab (or overlapping CI run) can trip | |
| # TooManyAssignmentsError transiently. Retry a few times before | |
| # giving up and needing a human to re-trigger. | |
| for attempt in 1 2 3; do | |
| if colab --auth=adc new -s "$SESSION" --gpu T4; then | |
| exit 0 | |
| fi | |
| echo "::warning::colab new attempt $attempt failed (likely GPU contention); retrying in 60s..." | |
| sleep 60 | |
| done | |
| echo "::error::colab new failed after 3 attempts" | |
| exit 1 | |
| - name: Write HuggingFace token (optional, improves download stability) | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| if [ -z "$HF_TOKEN" ]; then | |
| echo "::warning::HF_TOKEN secret not set — model downloads will be unauthenticated and may be slower or rate-limited. See README." | |
| else | |
| printf '%s' "$HF_TOKEN" > hf_token.txt | |
| fi | |
| - name: Upload shared inputs (ref audio, helper module, HF token) | |
| run: | | |
| SESSION="${{ steps.colab_new.outputs.session }}" | |
| colab --auth=adc upload -s "$SESSION" assets/ref_voice.wav /content/ref.wav | |
| colab --auth=adc upload -s "$SESSION" colab_job/_common.py /content/_common.py | |
| colab --auth=adc upload -s "$SESSION" colab_job/_synth_inner_25.py /content/_synth_inner_25.py | |
| colab --auth=adc upload -s "$SESSION" model_version.txt /content/model_version.txt | |
| if [ -f hf_token.txt ]; then | |
| colab --auth=adc upload -s "$SESSION" hf_token.txt /content/hf_token | |
| fi | |
| - name: Build environment and download models (once per issue) | |
| run: | | |
| set -o pipefail | |
| SESSION="${{ steps.colab_new.outputs.session }}" | |
| # colab exec's default --timeout is 30s, meant for short interactive | |
| # cells. setup.py enforces its own per-step timeouts internally | |
| # (git clone/uv sync/model download/check) and prints periodic | |
| # heartbeats; this outer value is just the overall safety net. | |
| # | |
| # google-colab-cli 0.6.0's `exec` never inspects the executed | |
| # code's own outcome -- it only fails on a *local* exception (e.g. | |
| # this outer --timeout firing). A script-side sys.exit(1) inside | |
| # the remote kernel is silently swallowed and `colab exec` still | |
| # exits 0. Confirmed on 2026-08-16: setup.py exhausted all 3 | |
| # download retries, printed "FATAL: ...", and the step still | |
| # reported success -- the run went on to "synthesize" against a | |
| # session with no model weights on disk. Until upstream fixes | |
| # this, grep the captured output for our own FATAL marker. | |
| colab --auth=adc exec -s "$SESSION" -f colab_job/setup.py --timeout 3000 \ | |
| 2>&1 | tee /tmp/exec_setup.log | |
| if grep -q '^FATAL:' /tmp/exec_setup.log; then | |
| echo "::error::setup.py reported FATAL (see log above) -- colab exec masked this as success" | |
| exit 1 | |
| fi | |
| - name: Synthesize each chunk on the warm session | |
| run: | | |
| set -o pipefail | |
| SESSION="${{ steps.colab_new.outputs.session }}" | |
| N=$(cat chunk_count.txt) | |
| echo "synthesizing $N chunk(s)" | |
| # 2026-08-17 real failure: a 9-chunk run (~55min in) died on the | |
| # LAST chunk's upload with "File or directory not found: | |
| # /content/...". Traced into colab_cli/contents.py: that message | |
| # is what it raises on an HTTP 404 from the Colab backend's | |
| # Contents API -- i.e. the session/kernel had gone away, not a | |
| # missing local file. A single transient hiccup this late costs | |
| # the entire run's already-completed work (the VM is stateful | |
| # but ephemeral -- nothing survives to resume from). Retry | |
| # uploads a few times before giving up. | |
| upload_with_retry () { | |
| local local_path="$1" remote_path="$2" attempt | |
| for attempt in 1 2 3; do | |
| if colab --auth=adc upload -s "$SESSION" "$local_path" "$remote_path"; then | |
| return 0 | |
| fi | |
| echo "::warning::upload of $local_path attempt $attempt failed; retrying in 20s..." | |
| sleep 20 | |
| done | |
| echo "::error::upload of $local_path failed after 3 attempts -- session likely died" | |
| return 1 | |
| } | |
| for i in $(seq 0 $((N - 1))); do | |
| echo "=== chunk $i/$((N - 1)) ===" | |
| echo -n "$i" > chunk_index.txt | |
| upload_with_retry chunk_index.txt /content/chunk_index.txt | |
| upload_with_retry "batch_chunk_$i.jsonl" "/content/batch_chunk_$i.jsonl" | |
| # Each call only pays model-load + that chunk's synth time (env | |
| # and model weights are already on disk from the setup step). | |
| # 1800s covers CHUNK_MAX_CHARS=700 with margin; see | |
| # scripts/parse_issue.py for the throughput math. | |
| # See the FATAL-grep note in the setup step above -- same | |
| # colab-exec exit-code gap applies here. | |
| colab --auth=adc exec -s "$SESSION" -f colab_job/synth_chunk.py --timeout 1800 \ | |
| 2>&1 | tee "/tmp/exec_chunk_$i.log" | |
| if grep -q '^FATAL:' "/tmp/exec_chunk_$i.log"; then | |
| echo "::error::synth_chunk.py failed on chunk $i (see log above)" | |
| exit 1 | |
| fi | |
| done | |
| - name: Concatenate chunks and download result | |
| run: | | |
| set -o pipefail | |
| SESSION="${{ steps.colab_new.outputs.session }}" | |
| # Same rationale as the retry wrapper in the chunk loop above -- | |
| # this runs after all chunks are done, so a transient upload | |
| # failure here would waste that entire completed run too. | |
| for attempt in 1 2 3; do | |
| colab --auth=adc upload -s "$SESSION" chunk_count.txt /content/chunk_count.txt && break | |
| echo "::warning::chunk_count.txt upload attempt $attempt failed; retrying in 20s..." | |
| sleep 20 | |
| [ "$attempt" = 3 ] && { echo "::error::chunk_count.txt upload failed after 3 attempts"; exit 1; } | |
| done | |
| colab --auth=adc exec -s "$SESSION" -f colab_job/concat_chunks.py --timeout 120 \ | |
| 2>&1 | tee /tmp/exec_concat.log | |
| if grep -q '^FATAL:' /tmp/exec_concat.log; then | |
| echo "::error::concat_chunks.py failed (see log above)" | |
| exit 1 | |
| fi | |
| colab --auth=adc download -s "$SESSION" /content/output.wav ./output.wav | |
| ls -lh output.wav | |
| - name: Stop Colab session | |
| if: always() | |
| run: | | |
| SESSION="${{ steps.colab_new.outputs.session }}" | |
| colab --auth=adc stop -s "$SESSION" || true | |
| - name: Create Release with audio | |
| if: success() | |
| id: release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "issues" ]; then | |
| TAG="issue-${{ github.event.issue.number }}-run${{ github.run_number }}" | |
| TITLE="將進酒朗讀 — issue #${{ github.event.issue.number }}" | |
| else | |
| TAG="manual-run${{ github.run_number }}" | |
| TITLE="將進酒朗讀 — manual run ${{ github.run_number }}" | |
| fi | |
| echo "tag=$TAG" >> "$GITHUB_OUTPUT" | |
| gh release create "$TAG" output.wav \ | |
| --title "$TITLE" \ | |
| --notes "Generated by IndexTTS-2 on a Colab T4 GPU. Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Comment result on issue | |
| if: success() && github.event_name == 'issues' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh issue comment ${{ github.event.issue.number }} --body \ | |
| "🎧 合成完成：${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ steps.release.outputs.tag }}" | |
| gh issue close ${{ github.event.issue.number }} | |
| - name: Comment failure on issue | |
| if: failure() && github.event_name == 'issues' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh issue comment ${{ github.event.issue.number }} --body \ | |
| "❌ 合成失敗，詳細記錄： ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| 常見原因：Colab 免費帳號同時只能有一個 GPU runtime；如果有人正在用瀏覽器開著另一個 Colab notebook 占用 T4，這裡就會失敗。關掉那個分頁後，開一個新的 issue 重試（workflow 只在 issue **開啟**時觸發，編輯這則 issue 不會重新跑）。" | |
| # GitHub Actions' concurrency group only holds the running run plus one | |
| # queued run -- if a third request comes in while this one is queued, | |
| # this run gets silently cancelled (not "failed", so `if: failure()` | |
| # above never fires) to make room for the newer one. Without this | |
| # step the issue just sits open forever with zero explanation. | |
| - name: Comment cancellation on issue | |
| if: cancelled() && github.event_name == 'issues' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh issue comment ${{ github.event.issue.number }} --body \ | |
| "⏸️ 這個請求被取消了。Workflow 現在用 \`queue: max\` 讓最多 100 個請求真的排隊等待，所以會走到這一步通常是排隊人數真的超過 100、或有人手動取消了這個 run——不是單純被下一個請求擠掉。開一個新的 issue 重新送出即可。" |