Add visualizer@v1 with beats, peaks, pitch, and frequency (#259)

Upgrades the TUI visualizer from the old `_draft_r1` draft to the newer
`visualizer@v1` role, rendering everything the server streams
time-aligned to the playhead: a beats strip with estimated BPM and
downbeats, a peaks strip of energy onsets, and pitch and
dominant-frequency (`f_peak`) cursors over the spectrum.

Only data that is sent by the server is displayed. For example, Music
Assistant 2.9 does not support pitch.

Author: maximmaxim345

README.md

@@ -155,6 +155,7 @@ Settings are stored in `~/.config/sendspin/`:
 | `manufacturer` | string | TUI/daemon | Manufacturer name reported in the client hello (`--manufacturer`) |
 | `product_name` | string | TUI/daemon | Product name reported in the client hello (`--product-name`); defaults to auto-detected OS/platform name |
 | `interface` | string | TUI/daemon | IP address of the network interface to use (`--interface`) |
+| `visualizer` | boolean | TUI | Render the `visualizer@v1` audio visualizer on launch (default: false). Toggle with `v` in the TUI |
 | `source` | string | serve | Default audio source (file path or URL, ffmpeg input) |
 | `source_format` | string | serve | ffmpeg container format for audio source |
 | `clients` | array | serve | Client URLs to connect to (`--client`) |
@@ -321,9 +322,17 @@ Hooks receive these environment variables:
 
 ### Visualizer
 
-The TUI includes a real-time audio spectrum visualizer that displays frequency data received from the server. This uses the experimental `visualizer@_draft_r1` role. The spectrum data is computed on the server and sent via sendspin to the TUI.
+The TUI includes a real-time audio visualizer driven by the `visualizer@v1` role. All analysis is computed on the server and streamed to the TUI, time-aligned to the audio playhead. It shows:
 
-Toggle it by pressing `v` in the TUI. Your preference is saved in settings and remembered on next launch.
+- **Spectrum bars** — frequency magnitude across the range, tinted by overall loudness (and by the album-artwork palette when the server provides one).
+- **Beats timeline** — a `beats (NNN BPM):` strip with the estimated tempo; downbeats render differently from regular beats.
+- **Peaks timeline** — a `peaks:` strip of energy onsets (transients like drum hits), independent of the beat grid, with glyph height scaled by onset strength.
+- **Pitch** — the perceived musical note (e.g. `A4`) with an arrow pointing at its position on the spectrum, shown whenever a pitch is detected.
+- **Dominant frequency** — an `f_peak:` readout with an arrow marking the loudest frequency on the spectrum.
+
+Lower rows are dropped first on short terminals, keeping the spectrum visible.
+
+Toggle the visualizer by pressing `v` in the TUI. Your preference is saved in settings and remembered on next launch.
 
 ### Debugging & Troubleshooting
 

pyproject.toml

@@ -14,7 +14,7 @@ classifiers = [
   "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-  "aiosendspin[server]~=5.3",
+  "aiosendspin[server]~=6.0.1",
   "aiosendspin-mpris~=2.1.1",
   "av>=15.0.0",
   "numpy>=1.26.0",

sendspin/tui/app.py

@@ -22,15 +22,18 @@
     GroupUpdateServerPayload,
     ServerCommandPayload,
     ServerStatePayload,
+    StreamStartMessage,
 )
 from aiosendspin.models.player import (
     ClientHelloPlayerSupport,
     PlayerCommandPayload,
     SupportedAudioFormat,
 )
 from aiosendspin.models.visualizer import (
+    BeatTiming,
     ClientHelloVisualizerSpectrum,
     ClientHelloVisualizerSupport,
+    StreamStartVisualizer,
     VisualizerFrame,
 )
 from aiosendspin.models.types import (
@@ -49,8 +52,15 @@
 from sendspin.settings import ClientSettings
 from sendspin.tui.keyboard import keyboard_loop
 from sendspin.tui.ui import ColorMode, SendspinUI
+from sendspin.tui.visualizer import (
+    SPECTRUM_F_MAX,
+    SPECTRUM_F_MIN,
+    SPECTRUM_N_BINS,
+    SPECTRUM_SCALE,
+    PeakEvent,
+)
 from sendspin.utils import create_task, get_device_info
-from sendspin.visualizer_connector import VisualizerHandler
+from sendspin.visualizer_connector import BeatHandler, PeakHandler, VisualizerHandler
 
 logger = logging.getLogger(__name__)
 
@@ -251,6 +261,8 @@ def __init__(self, args: AppArgs) -> None:
         self._client: SendspinClient | None = None
         self._audio_handler: AudioStreamHandler | None = None
         self._visualizer_handler: VisualizerHandler | None = None
+        self._beat_handler: BeatHandler | None = None
+        self._peak_handler: PeakHandler | None = None
         self._settings = args.settings
         self._visualizer_enabled: bool = args.settings.visualizer
         # Currently-applied static delay in milliseconds, mirroring
@@ -267,17 +279,16 @@ def __init__(self, args: AppArgs) -> None:
 
     @staticmethod
     def _build_visualizer_support() -> ClientHelloVisualizerSupport:
-        """Build visualizer support payload for client/hello."""
+        """Build visualizer support payload for client/hello (visualizer@v1)."""
         return ClientHelloVisualizerSupport(
             buffer_capacity=65536,
-            types=["loudness", "spectrum"],
-            batch_max=8,
+            types=["loudness", "spectrum", "beat", "peak", "f_peak", "pitch"],
+            rate_max=30,
             spectrum=ClientHelloVisualizerSpectrum(
-                n_disp_bins=48,
-                scale="mel",
-                f_min=20,
-                f_max=20000,
-                rate_max=30,
+                n_disp_bins=SPECTRUM_N_BINS,
+                scale=SPECTRUM_SCALE,
+                f_min=SPECTRUM_F_MIN,
+                f_max=SPECTRUM_F_MAX,
             ),
         )
 
@@ -331,6 +342,21 @@ def _attach_client(self) -> None:
                 on_frame=self._handle_visualizer_frame,
             )
             self._visualizer_handler.attach_client(self._client)
+            self._beat_handler = BeatHandler(
+                on_beat=self._handle_beat,
+                on_schedule=self._handle_beat_schedule,
+            )
+            self._beat_handler.attach_client(self._client)
+            self._peak_handler = PeakHandler(
+                on_peak=self._handle_peak,
+                on_schedule=self._handle_peak_schedule,
+            )
+            self._peak_handler.attach_client(self._client)
+            self._listener_unsubscribes.append(
+                self._client.add_stream_start_listener(self._handle_stream_start)
+            )
+            if self._ui is not None:
+                self._ui.set_server_clock(self._server_now_us)
 
         if MPRIS_AVAILABLE and self._args.use_mpris:
             self._mpris = SendspinMpris(self._client)
@@ -351,6 +377,15 @@ def _detach_client(self) -> None:
         if self._visualizer_handler:
             self._visualizer_handler.detach()
             self._visualizer_handler = None
+        if self._beat_handler:
+            self._beat_handler.detach()
+            self._beat_handler = None
+        if self._peak_handler:
+            self._peak_handler.detach()
+            self._peak_handler = None
+        if self._ui is not None:
+            self._ui.set_server_clock(None)
+            self._ui.set_visualizer_types(frozenset())
 
         if self._mpris:
             self._mpris.stop()
@@ -492,6 +527,10 @@ def signal_handler() -> None:
                 self._mpris.stop()
             if self._visualizer_handler:
                 self._visualizer_handler.detach()
+            if self._beat_handler:
+                self._beat_handler.detach()
+            if self._peak_handler:
+                self._peak_handler.detach()
             if self._ui:
                 self._ui.stop()
             if self._audio_handler:
@@ -524,6 +563,7 @@ async def _handle_disconnect(self, message: str) -> None:
         self._ui.set_disconnected(message)
         if self._visualizer_handler:
             self._visualizer_handler.reset()
+        self._clear_visualizer_timelines()
         await self._audio_handler.handle_disconnect()
 
     async def _connect_cancellable(self, url: str) -> None:
@@ -718,6 +758,16 @@ def _handle_metadata_update(self, payload: ServerStatePayload) -> None:
             ui.set_repeat_shuffle(state.repeat_mode, state.shuffle)
         ui.add_event(state.describe())
 
+    def _clear_visualizer_timelines(self) -> None:
+        """Drop the beat and peak strips, cancelling their pending schedules."""
+        if self._beat_handler:
+            self._beat_handler.reset()
+        if self._peak_handler:
+            self._peak_handler.reset()
+        if self._ui is not None:
+            self._ui.clear_beats()
+            self._ui.clear_peaks()
+
     def _handle_color_update(self, payload: ServerStatePayload) -> None:
         """Forward a color@v1 palette payload to the UI."""
         assert self._ui is not None
@@ -744,9 +794,13 @@ def _handle_group_update(self, payload: GroupUpdateServerPayload) -> None:
         with ui.batch_update():
             ui.set_group_name(payload.group_name)
             if payload.playback_state:
+                changed = payload.playback_state != state.playback_state
                 state.playback_state = payload.playback_state
                 ui.set_playback_state(payload.playback_state)
                 ui.add_event(f"Playback state: {payload.playback_state.value}")
+                # No stream/clear fires on pause, so drop the frozen strips here.
+                if changed and payload.playback_state != PlaybackStateType.PLAYING:
+                    self._clear_visualizer_timelines()
 
     def _handle_server_state(self, payload: ServerStatePayload) -> None:
         """Handle server/state messages with controller state."""
@@ -840,10 +894,57 @@ async def _toggle_visualizer(self) -> None:
             if not self._cancel_connect():
                 await old_client.disconnect()
 
+    def _server_now_us(self) -> int:
+        """Current time in the server clock the beat and peak strips are drawn against.
+
+        Strip events carry server-clock timestamps, so the strip's "now" must be
+        in the same domain. This is the inverse of ``compute_play_time``, so an
+        event lands on the playhead cell exactly when it becomes audible.
+        """
+        assert self._client is not None
+        return self._client.compute_server_time(self._client.now_us())
+
+    def _handle_stream_start(self, message: StreamStartMessage) -> None:
+        """Record which visualizer types the server negotiated for this stream."""
+        if self._ui is None:
+            return
+        config = message.payload.visualizer
+        types = (
+            frozenset(config.types) if isinstance(config, StreamStartVisualizer) else frozenset()
+        )
+        self._ui.set_visualizer_types(types)
+
     def _handle_visualizer_frame(self, frame: VisualizerFrame) -> None:
         """Handle a visualizer frame from the connector."""
         if self._ui is not None:
-            self._ui.set_visualizer_frame(frame.spectrum, frame.loudness)
+            self._ui.set_visualizer_frame(
+                frame.spectrum,
+                frame.loudness,
+                frame.pitch_midi_q88,
+                frame.f_peak_freq,
+            )
+
+    def _handle_beat(self, beat: BeatTiming) -> None:
+        """Handle a beat hitting the playhead."""
+        if self._ui is not None:
+            self._ui.record_beat(beat)
+
+    def _handle_beat_schedule(self, scheduled: list[BeatTiming]) -> None:
+        """Handle an updated upcoming-beat schedule."""
+        if self._ui is not None:
+            self._ui.set_beat_schedule(scheduled)
+
+    def _handle_peak(self, timestamp_us: int, strength: int) -> None:
+        """Handle an energy-onset peak hitting the playhead."""
+        if self._ui is not None:
+            self._ui.record_peak(PeakEvent(timestamp_us=timestamp_us, strength=strength))
+
+    def _handle_peak_schedule(self, scheduled: list[tuple[int, int]]) -> None:
+        """Handle an updated upcoming-peak schedule."""
+        if self._ui is not None:
+            self._ui.set_peak_schedule(
+                [PeakEvent(timestamp_us=ts, strength=strength) for ts, strength in scheduled]
+            )
 
     def _on_stream_event(self, event: str) -> None:
         """Handle stream lifecycle events by running hooks."""