feat: add Whisper local transcription fallback for lyrics (#5)

When syncedlyrics providers find no lyrics, fall back to local Whisper
transcription to generate timestamped .lrc files from the audio itself.

- Lazy-loads Whisper model on first use (configurable: tiny/base/small/medium/large)
- Converts Whisper segments to LRC format with timestamps
- New config options: whisper_fallback (default: true), whisper_model (default: base)
- 13 lyrics tests (4 new for Whisper fallback)
- Bump to v0.4.0

Author: GeiserX

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.13-slim
+FROM python:3.14-slim
 
 # Install FFmpeg for transcoding
 RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "audio-transcode-watcher"
-version = "0.3.0"
+version = "0.4.0"
 description = "Watch a source folder and automatically transcode audio files to multiple formats"
 readme = "README.md"
 license = "GPL-3.0-or-later"
@@ -28,6 +28,7 @@ dependencies = [
     "pyyaml>=6.0",
     "mutagen>=1.47.0",
     "syncedlyrics>=1.0.0",
+    "openai-whisper>=20240930",
 ]
 
 [project.optional-dependencies]

src/audio_transcode_watcher/config.py

@@ -94,6 +94,8 @@ class Config:
     stability_timeout: float = 60.0
     min_stable_seconds: float = 1.0
     fetch_lyrics: bool = True  # Auto-fetch .lrc lyrics via syncedlyrics
+    whisper_fallback: bool = True  # Use Whisper local transcription as fallback
+    whisper_model: str = "base"  # Whisper model size: tiny, base, small, medium, large
 
     def __post_init__(self) -> None:
         """Validate configuration after initialization."""
@@ -140,6 +142,8 @@ def from_dict(cls, data: dict[str, Any]) -> Config:
             stability_timeout=settings.get("stability_timeout", 60.0),
             min_stable_seconds=settings.get("min_stable_seconds", 1.0),
             fetch_lyrics=settings.get("fetch_lyrics", True),
+            whisper_fallback=settings.get("whisper_fallback", True),
+            whisper_model=settings.get("whisper_model", "base"),
         )
 
     @classmethod

src/audio_transcode_watcher/lyrics.py

@@ -1,4 +1,4 @@
-"""Automatic lyrics fetching for audio files using syncedlyrics."""
+"""Automatic lyrics fetching with syncedlyrics and Whisper fallback."""
 
 from __future__ import annotations
 
@@ -14,6 +14,76 @@
 
 logger = logging.getLogger(__name__)
 
+# Lazy-loaded Whisper model (heavyweight, only load once when needed)
+_whisper_model = None
+_whisper_load_failed = False
+
+
+def _get_whisper_model(model_name: str = "base"):
+    """Lazy-load and cache the Whisper model."""
+    global _whisper_model, _whisper_load_failed
+    if _whisper_load_failed:
+        return None
+    if _whisper_model is not None:
+        return _whisper_model
+    try:
+        import whisper
+
+        logger.info("Loading Whisper model '%s' (first use, may take a moment)...", model_name)
+        _whisper_model = whisper.load_model(model_name)
+        logger.info("Whisper model '%s' loaded", model_name)
+        return _whisper_model
+    except Exception:
+        logger.warning("Failed to load Whisper model", exc_info=True)
+        _whisper_load_failed = True
+        return None
+
+
+def _segments_to_lrc(segments: list[dict]) -> str:
+    """Convert Whisper transcript segments to LRC format."""
+    lines = []
+    for seg in segments:
+        start = seg.get("start", 0.0)
+        text = seg.get("text", "").strip()
+        if not text:
+            continue
+        mins = int(start // 60)
+        secs = start % 60
+        lines.append(f"[{mins:02d}:{secs:05.2f}] {text}")
+    return "\n".join(lines)
+
+
+def _transcribe_with_whisper(filepath: str, model_name: str = "base") -> str | None:
+    """
+    Transcribe audio to synced lyrics using Whisper.
+
+    Args:
+        filepath: Path to the audio file.
+        model_name: Whisper model size (tiny, base, small, medium, large).
+
+    Returns:
+        LRC-formatted string, or None on failure.
+    """
+    model = _get_whisper_model(model_name)
+    if model is None:
+        return None
+
+    try:
+        logger.info("Transcribing with Whisper: %s", Path(filepath).name)
+        result = model.transcribe(filepath, verbose=False)
+        segments = result.get("segments", [])
+        if not segments:
+            logger.info("Whisper produced no segments for: %s", Path(filepath).name)
+            return None
+        lrc = _segments_to_lrc(segments)
+        logger.info(
+            "Whisper transcribed %d segments for: %s", len(segments), Path(filepath).name
+        )
+        return lrc
+    except Exception:
+        logger.warning("Whisper transcription failed for: %s", filepath, exc_info=True)
+        return None
+
 
 def extract_metadata(filepath: str) -> tuple[str, str] | None:
     """
@@ -53,15 +123,23 @@ def extract_metadata(filepath: str) -> tuple[str, str] | None:
     return None
 
 
-def fetch_lyrics_for_file(filepath: str) -> str | None:
+def fetch_lyrics_for_file(
+    filepath: str,
+    whisper_fallback: bool = True,
+    whisper_model: str = "base",
+) -> str | None:
     """
     Fetch synced lyrics (.lrc) for an audio file if not already present.
 
-    Checks if an .lrc sidecar already exists.  If not, extracts metadata
-    and queries syncedlyrics providers (Musixmatch, LRCLIB, NetEase, etc.).
+    Strategy:
+      1. Check if .lrc sidecar already exists -> skip
+      2. Try syncedlyrics providers (Musixmatch, LRCLIB, NetEase)
+      3. If nothing found and whisper_fallback enabled, transcribe locally
 
     Args:
         filepath: Path to the audio file.
+        whisper_fallback: Use Whisper local transcription as fallback.
+        whisper_model: Whisper model size (tiny, base, small, medium, large).
 
     Returns:
         Path to the written .lrc file, or None if lyrics were not found
@@ -74,28 +152,41 @@ def fetch_lyrics_for_file(filepath: str) -> str | None:
     if os.path.isfile(lrc_path):
         return None
 
+    lrc_content: str | None = None
+
+    # Step 1: Try syncedlyrics
     meta = extract_metadata(filepath)
-    if meta is None:
+    if meta is not None:
+        artist, title = meta
+        query = f"{artist} {title}"
+        try:
+            lrc_content = syncedlyrics.search(query)
+        except Exception:
+            logger.warning("syncedlyrics search failed for: %s", query, exc_info=True)
+
+        if lrc_content:
+            return _write_lrc(lrc_path, lrc_content, f"{artist} - {title}", "syncedlyrics")
+
+        logger.info("No lyrics found via syncedlyrics for: %s - %s", artist, title)
+    else:
         logger.debug("Cannot extract metadata for lyrics: %s", filepath)
-        return None
 
-    artist, title = meta
-    query = f"{artist} {title}"
+    # Step 2: Whisper fallback
+    if whisper_fallback:
+        lrc_content = _transcribe_with_whisper(filepath, whisper_model)
+        if lrc_content:
+            label = f"{meta[0]} - {meta[1]}" if meta else Path(filepath).stem
+            return _write_lrc(lrc_path, lrc_content, label, "whisper")
 
-    try:
-        lrc_content = syncedlyrics.search(query)
-    except Exception:
-        logger.warning("syncedlyrics search failed for: %s", query, exc_info=True)
-        return None
+    return None
 
-    if not lrc_content:
-        logger.info("No lyrics found for: %s - %s", artist, title)
-        return None
 
+def _write_lrc(lrc_path: str, content: str, label: str, source: str) -> str | None:
+    """Write LRC content to disk."""
     try:
         with open(lrc_path, "w", encoding="utf-8") as f:
-            f.write(lrc_content)
-        logger.info("♫ lyrics saved: %s - %s → %s", artist, title, lrc_path)
+            f.write(content)
+        logger.info("♫ lyrics saved (%s): %s → %s", source, label, lrc_path)
         return lrc_path
     except Exception:
         logger.error("Failed to write lyrics file: %s", lrc_path, exc_info=True)

src/audio_transcode_watcher/sync.py

@@ -122,7 +122,11 @@ def process_source_file(
         # Auto-fetch lyrics if enabled and no .lrc sidecar exists
         if config.fetch_lyrics:
             try:
-                fetch_lyrics_for_file(source_path)
+                fetch_lyrics_for_file(
+                    source_path,
+                    whisper_fallback=config.whisper_fallback,
+                    whisper_model=config.whisper_model,
+                )
             except Exception:
                 logger.debug("Lyrics fetch failed for %s", source_path, exc_info=True)
         sync_sidecars(source_path, config)

tests/test_lyrics.py

@@ -7,7 +7,11 @@
 
 import pytest
 
-from audio_transcode_watcher.lyrics import extract_metadata, fetch_lyrics_for_file
+from audio_transcode_watcher.lyrics import (
+    _segments_to_lrc,
+    extract_metadata,
+    fetch_lyrics_for_file,
+)
 
 
 class TestExtractMetadata:
@@ -57,6 +61,28 @@ def test_falls_back_to_filename_on_empty_tags(self, mock_mutagen, tmp_path):
         assert result == ("AC DC", "Thunderstruck")
 
 
+class TestSegmentsToLrc:
+    """Tests for _segments_to_lrc()."""
+
+    def test_converts_segments(self):
+        segments = [
+            {"start": 0.0, "text": "Hello world"},
+            {"start": 65.5, "text": "Second line"},
+        ]
+        result = _segments_to_lrc(segments)
+        assert "[00:00.00] Hello world" in result
+        assert "[01:05.50] Second line" in result
+
+    def test_skips_empty_text(self):
+        segments = [
+            {"start": 0.0, "text": "  "},
+            {"start": 5.0, "text": "Real line"},
+        ]
+        result = _segments_to_lrc(segments)
+        assert "Real line" in result
+        assert result.count("[") == 1
+
+
 class TestFetchLyricsForFile:
     """Tests for fetch_lyrics_for_file()."""
 
@@ -78,28 +104,60 @@ def test_fetches_and_writes_lyrics(self, mock_meta, mock_syncedlyrics, tmp_path)
         mock_meta.return_value = ("Queen", "Radio Gaga")
         mock_syncedlyrics.search.return_value = "[00:01.00] All we hear is\n[00:03.00] Radio gaga"
 
-        result = fetch_lyrics_for_file(str(audio))
+        result = fetch_lyrics_for_file(str(audio), whisper_fallback=False)
         assert result is not None
         assert result.endswith(".lrc")
         assert os.path.isfile(result)
         content = open(result).read()
         assert "Radio gaga" in content
 
+    @patch("audio_transcode_watcher.lyrics._transcribe_with_whisper")
+    @patch("audio_transcode_watcher.lyrics.syncedlyrics")
+    @patch("audio_transcode_watcher.lyrics.extract_metadata")
+    def test_falls_back_to_whisper(self, mock_meta, mock_syncedlyrics, mock_whisper, tmp_path):
+        """Fall back to Whisper when syncedlyrics returns nothing."""
+        audio = tmp_path / "Niche Band - Rare Song.flac"
+        audio.touch()
+        mock_meta.return_value = ("Niche Band", "Rare Song")
+        mock_syncedlyrics.search.return_value = None
+        mock_whisper.return_value = "[00:00.00] Transcribed by whisper"
+
+        result = fetch_lyrics_for_file(str(audio), whisper_fallback=True)
+        assert result is not None
+        assert os.path.isfile(result)
+        content = open(result).read()
+        assert "Transcribed by whisper" in content
+        mock_whisper.assert_called_once()
+
+    @patch("audio_transcode_watcher.lyrics._transcribe_with_whisper")
+    @patch("audio_transcode_watcher.lyrics.syncedlyrics")
+    @patch("audio_transcode_watcher.lyrics.extract_metadata")
+    def test_whisper_disabled(self, mock_meta, mock_syncedlyrics, mock_whisper, tmp_path):
+        """Don't use Whisper when disabled."""
+        audio = tmp_path / "Artist - Song.flac"
+        audio.touch()
+        mock_meta.return_value = ("Artist", "Song")
+        mock_syncedlyrics.search.return_value = None
+
+        result = fetch_lyrics_for_file(str(audio), whisper_fallback=False)
+        assert result is None
+        mock_whisper.assert_not_called()
+
     @patch("audio_transcode_watcher.lyrics.syncedlyrics")
     @patch("audio_transcode_watcher.lyrics.extract_metadata")
     def test_returns_none_when_no_lyrics(self, mock_meta, mock_syncedlyrics, tmp_path):
-        """Return None when syncedlyrics finds nothing."""
+        """Return None when nothing is found."""
         audio = tmp_path / "Obscure Band - Niche Song.flac"
         audio.touch()
         mock_meta.return_value = ("Obscure Band", "Niche Song")
         mock_syncedlyrics.search.return_value = None
 
-        result = fetch_lyrics_for_file(str(audio))
+        result = fetch_lyrics_for_file(str(audio), whisper_fallback=False)
         assert result is None
 
     def test_returns_none_when_no_metadata(self, tmp_path):
         """Return None when metadata can't be extracted."""
         audio = tmp_path / "noinfo.flac"
         audio.touch()
-        result = fetch_lyrics_for_file(str(audio))
+        result = fetch_lyrics_for_file(str(audio), whisper_fallback=False)
         assert result is None