From 8d5c936de577965a407427cc6ba5c77531d6072f Mon Sep 17 00:00:00 2001 From: Juan Pablo Farias Date: Fri, 4 Sep 2026 15:34:49 -0300 Subject: [PATCH 1/6] feat: introduce MACOS_SDK conditional compilation --- GalaxyBudsClient/App.axaml.cs | 4 +++- GalaxyBudsClient/GalaxyBudsClient.csproj | 5 +++-- GalaxyBudsClient/Platform.props | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/GalaxyBudsClient/App.axaml.cs b/GalaxyBudsClient/App.axaml.cs index 212c737fc..94d402c20 100644 --- a/GalaxyBudsClient/App.axaml.cs +++ b/GalaxyBudsClient/App.axaml.cs @@ -1,4 +1,4 @@ -#if OSX +#if OSX && MACOS_SDK using AppKit; #endif using System; @@ -66,7 +66,9 @@ public override void Initialize() DataContext = this; #if OSX +#if MACOS_SDK NSApplication.Init(); +#endif // For menu bar applications (LSUIElement=true), hide the dock icon immediately at startup. // The dock icon will only appear when the settings window is explicitly opened. GalaxyBudsClient.Platform.OSX.AppUtils.setHideInDock(true); diff --git a/GalaxyBudsClient/GalaxyBudsClient.csproj b/GalaxyBudsClient/GalaxyBudsClient.csproj index 5836d3529..a4ee2209b 100644 --- a/GalaxyBudsClient/GalaxyBudsClient.csproj +++ b/GalaxyBudsClient/GalaxyBudsClient.csproj @@ -1,4 +1,4 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml.cs b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml.cs new file mode 100644 index 000000000..fad4c9c8d --- /dev/null +++ b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml.cs @@ -0,0 +1,11 @@ +using GalaxyBudsClient.Interface.ViewModels.Pages; + +namespace GalaxyBudsClient.Interface.Pages; + +public partial class SpatialAudioPage : BasePage +{ + public SpatialAudioPage() + { + InitializeComponent(); + } +} diff --git a/GalaxyBudsClient/Interface/ViewModels/Pages/SpatialAudioPageViewModel.cs b/GalaxyBudsClient/Interface/ViewModels/Pages/SpatialAudioPageViewModel.cs new file mode 100644 index 000000000..4d97aaeb0 --- /dev/null +++ b/GalaxyBudsClient/Interface/ViewModels/Pages/SpatialAudioPageViewModel.cs @@ -0,0 +1,147 @@ +using System; +using System.ComponentModel; +using Avalonia.Controls; +using Avalonia.Threading; +using FluentIcons.Common; +using GalaxyBudsClient.Generated.I18N; +using GalaxyBudsClient.Interface.Pages; +using GalaxyBudsClient.Model; +using GalaxyBudsClient.Platform.SpatialAudio; +using ReactiveUI.SourceGenerators; + +namespace GalaxyBudsClient.Interface.ViewModels.Pages; + +public partial class SpatialAudioPageViewModel : MainPageViewModelBase, IDisposable +{ + private readonly OscSpatialBroadcaster _oscBroadcaster = new(); + private readonly OpenTrackBroadcaster _openTrackBroadcaster = new(); + private readonly BinauralDemoAudioPlayer _demoPlayer = new(); + + [Reactive] private bool _isTrackingEnabled; + [Reactive] private double _yaw; + [Reactive] private double _pitch; + [Reactive] private double _roll; + [Reactive] private bool _isOscBroadcasting; + [Reactive] private bool _isOpenTrackBroadcasting; + [Reactive] private bool _isDemoPlaying; + [Reactive] private string _statusText = Strings.SpatialTrackingInactive; + + public SpatialAudioPageViewModel() + { + SpatialAudioService.Instance.OrientationUpdated += OnOrientationUpdated; + SpatialAudioService.Instance.PropertyChanged += OnServicePropertyChanged; + _demoPlayer.PlaybackStateChanged += OnPlaybackStateChanged; + PropertyChanged += OnSelfPropertyChanged; + + IsTrackingEnabled = SpatialAudioService.Instance.IsActive; + UpdateStatusText(); + } + + private void OnServicePropertyChanged(object? sender, PropertyChangedEventArgs e) + { + if (e.PropertyName == nameof(SpatialAudioService.IsActive)) + { + Dispatcher.UIThread.Post(() => + { + IsTrackingEnabled = SpatialAudioService.Instance.IsActive; + UpdateStatusText(); + }); + } + } + + private void OnSelfPropertyChanged(object? sender, PropertyChangedEventArgs e) + { + switch (e.PropertyName) + { + case nameof(IsTrackingEnabled): + if (IsTrackingEnabled && !SpatialAudioService.Instance.IsActive) + { + SpatialAudioService.Instance.Start(); + } + else if (!IsTrackingEnabled && SpatialAudioService.Instance.IsActive) + { + SpatialAudioService.Instance.Stop(); + } + UpdateStatusText(); + break; + + case nameof(IsOscBroadcasting): + _oscBroadcaster.IsEnabled = IsOscBroadcasting; + break; + + case nameof(IsOpenTrackBroadcasting): + _openTrackBroadcaster.IsEnabled = IsOpenTrackBroadcasting; + break; + } + } + + private void OnOrientationUpdated(object? sender, SpatialOrientationEventArgs e) + { + Dispatcher.UIThread.Post(() => + { + Yaw = Math.Round(e.Yaw, 1); + Pitch = Math.Round(e.Pitch, 1); + Roll = Math.Round(e.Roll, 1); + }, DispatcherPriority.Render); + } + + private void OnPlaybackStateChanged(object? sender, bool isPlaying) + { + Dispatcher.UIThread.Post(() => + { + IsDemoPlaying = isPlaying; + }); + } + + public void Recenter() + { + SpatialAudioService.Instance.Recenter(); + } + + public async void ToggleDemo() + { + if (IsDemoPlaying) + { + _demoPlayer.Stop(); + } + else + { + if (!SpatialAudioService.Instance.IsActive) + { + IsTrackingEnabled = true; + } + await _demoPlayer.StartAsync(); + } + } + + private void UpdateStatusText() + { + StatusText = IsTrackingEnabled ? Strings.SpatialTrackingActive : Strings.SpatialTrackingInactive; + } + + public override void OnNavigatedFrom() + { + base.OnNavigatedFrom(); + if (IsDemoPlaying) + { + _demoPlayer.Stop(); + } + } + + public override Control CreateView() => new SpatialAudioPage { DataContext = this }; + + public override string TitleKey => Keys.PageSpatialAudio; + public override Symbol IconKey => Symbol.SoundWaveCircle; + public override bool ShowsInFooter => false; + + public void Dispose() + { + SpatialAudioService.Instance.OrientationUpdated -= OnOrientationUpdated; + SpatialAudioService.Instance.PropertyChanged -= OnServicePropertyChanged; + _demoPlayer.PlaybackStateChanged -= OnPlaybackStateChanged; + _demoPlayer.Dispose(); + _oscBroadcaster.Dispose(); + _openTrackBroadcaster.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/BinauralDemoAudioPlayer.cs b/GalaxyBudsClient/Platform/SpatialAudio/BinauralDemoAudioPlayer.cs new file mode 100644 index 000000000..c4559dce9 --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/BinauralDemoAudioPlayer.cs @@ -0,0 +1,204 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Serilog; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +/// +/// Interactive audio player for demonstrating 360 spatial audio anchored in front of the screen. +/// Simulates binaural hearing (ILD - Interaural Level Difference & ITD - Interaural Time Difference). +/// +public class BinauralDemoAudioPlayer : IDisposable +{ + private CancellationTokenSource? _playCts; + private bool _isPlaying; + + public bool IsPlaying + { + get => _isPlaying; + private set => _isPlaying = value; + } + + public event EventHandler? PlaybackStateChanged; + + public async Task StartAsync() + { + if (_isPlaying) + return; + + _isPlaying = true; + PlaybackStateChanged?.Invoke(this, true); + _playCts = new CancellationTokenSource(); + + try + { + await Task.Run(() => PlaybackLoop(_playCts.Token)); + } + catch (OperationCanceledException) + { + // Normal exit + } + catch (Exception ex) + { + Log.Error(ex, "BinauralDemoAudioPlayer: Playback error"); + } + finally + { + _isPlaying = false; + PlaybackStateChanged?.Invoke(this, false); + } + } + + public void Stop() + { + if (!_isPlaying) + return; + + _playCts?.Cancel(); + _playCts?.Dispose(); + _playCts = null; + _isPlaying = false; + PlaybackStateChanged?.Invoke(this, false); + } + + private void PlaybackLoop(CancellationToken token) + { + var tempFile = Path.Combine(Path.GetTempPath(), "gbc_spatial_demo.wav"); + + while (!token.IsCancellationRequested) + { + var yaw = SpatialAudioService.Instance.CurrentYaw; + + // Generate a 1.2-second spatial chime centered at the screen + GenerateSpatialChimeWav(tempFile, yaw); + + if (token.IsCancellationRequested) + break; + + PlayWavFile(tempFile, token); + + // Interval between chimes + try + { + Task.Delay(1300, token).Wait(token); + } + catch (OperationCanceledException) + { + break; + } + } + + try + { + if (File.Exists(tempFile)) + File.Delete(tempFile); + } + catch + { + // Ignore cleanup errors + } + } + + private static void PlayWavFile(string path, CancellationToken token) + { + try + { + if (OperatingSystem.IsMacOS()) + { + using var process = Process.Start(new ProcessStartInfo + { + FileName = "afplay", + Arguments = $"\"{path}\"", + CreateNoWindow = true, + UseShellExecute = false + }); + + if (process != null) + { + while (!process.WaitForExit(100)) + { + if (token.IsCancellationRequested) + { + process.Kill(); + break; + } + } + } + } + } + catch (Exception ex) + { + Log.Warning(ex, "BinauralDemoAudioPlayer: afplay failed"); + } + } + + private static void GenerateSpatialChimeWav(string filePath, float yawDegrees) + { + const int sampleRate = 44100; + const double durationSeconds = 1.0; + var totalSamples = (int)(sampleRate * durationSeconds); + + // Compute Interaural Level Difference (ILD) + // Sound source is fixed at 0° (screen). + // Head yaw: when head turns left (yaw < 0), sound arrives at right ear first and louder. + var angleRad = (yawDegrees * Math.PI) / 180.0; + var pan = Math.Sin(angleRad); // -1.0 (hard right in ear) to +1.0 (hard left in ear) + + // Equal power panning + var leftVol = Math.Cos((pan + 1.0) * Math.PI / 4.0); + var rightVol = Math.Sin((pan + 1.0) * Math.PI / 4.0); + + // Compute Interaural Time Difference (ITD): maximum human delay is ~0.65ms (~29 samples) + var maxDelaySamples = (int)(sampleRate * 0.00065); + var delaySamples = (int)(pan * maxDelaySamples); + var leftDelay = delaySamples > 0 ? delaySamples : 0; + var rightDelay = delaySamples < 0 ? -delaySamples : 0; + + using var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); + using var writer = new BinaryWriter(stream); + + // WAV Header + writer.Write("RIFF"u8); + writer.Write(36 + (totalSamples * 4)); + writer.Write("WAVEfmt "u8); + writer.Write(16); // Subchunk1Size (16 for PCM) + writer.Write((short)1); // AudioFormat (1 = PCM) + writer.Write((short)2); // NumChannels (2 = Stereo) + writer.Write(sampleRate); + writer.Write(sampleRate * 4); // ByteRate + writer.Write((short)4); // BlockAlign + writer.Write((short)16); // BitsPerSample + writer.Write("data"u8); + writer.Write(totalSamples * 4); + + // Harmonic spatial chime: 528Hz (C) + 792Hz (G) harmonic bell + for (var i = 0; i < totalSamples; i++) + { + var t = (double)i / sampleRate; + var envelope = Math.Exp(-4.5 * t); // Percussive decay + + var sample1 = Math.Sin(2.0 * Math.PI * 528.0 * t); + var sample2 = 0.5 * Math.Sin(2.0 * Math.PI * 792.0 * t); + var sample3 = 0.25 * Math.Sin(2.0 * Math.PI * 1056.0 * t); + var tone = (sample1 + sample2 + sample3) * envelope * 0.45; + + // Apply delay and volume per channel + var leftIdx = i - leftDelay; + var rightIdx = i - rightDelay; + + var leftSample = leftIdx >= 0 ? (short)(tone * leftVol * 32767.0) : (short)0; + var rightSample = rightIdx >= 0 ? (short)(tone * rightVol * 32767.0) : (short)0; + + writer.Write(leftSample); + writer.Write(rightSample); + } + } + + public void Dispose() + { + Stop(); + } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/OpenTrackBroadcaster.cs b/GalaxyBudsClient/Platform/SpatialAudio/OpenTrackBroadcaster.cs new file mode 100644 index 000000000..51d9b9013 --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/OpenTrackBroadcaster.cs @@ -0,0 +1,104 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Sockets; +using Serilog; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +/// +/// Broadcasts 6DOF head tracking data via standard UDP protocol for OpenTrack / FreeTrack (port 4242). +/// Enables head tracking in flight simulators, racing games, and space combat games. +/// +public class OpenTrackBroadcaster : IDisposable +{ + private UdpClient? _udpClient; + private IPEndPoint? _endpoint; + private bool _isEnabled; + + public bool IsEnabled + { + get => _isEnabled; + set + { + if (_isEnabled == value) return; + _isEnabled = value; + if (_isEnabled) + { + InitSocket(); + SpatialAudioService.Instance.OrientationUpdated += OnOrientationUpdated; + } + else + { + SpatialAudioService.Instance.OrientationUpdated -= OnOrientationUpdated; + CloseSocket(); + } + } + } + + public string Host { get; set; } = "127.0.0.1"; + public int Port { get; set; } = 4242; + + private void InitSocket() + { + try + { + _endpoint = new IPEndPoint(IPAddress.Parse(Host), Port); + _udpClient = new UdpClient(); + Log.Information("OpenTrackBroadcaster: Started broadcasting to {Host}:{Port}", Host, Port); + } + catch (Exception ex) + { + Log.Error(ex, "OpenTrackBroadcaster: Failed to initialize UDP client"); + _isEnabled = false; + } + } + + private void CloseSocket() + { + try + { + _udpClient?.Close(); + _udpClient?.Dispose(); + _udpClient = null; + _endpoint = null; + } + catch (Exception ex) + { + Log.Warning(ex, "OpenTrackBroadcaster: Error while closing socket"); + } + } + + private void OnOrientationUpdated(object? sender, SpatialOrientationEventArgs e) + { + if (!_isEnabled || _udpClient == null || _endpoint == null) + return; + + try + { + // OpenTrack standard protocol expects 6 double precision floats (48 bytes) in little endian: + // X (cm), Y (cm), Z (cm), Yaw (deg), Pitch (deg), Roll (deg) + using var stream = new MemoryStream(48); + using var writer = new BinaryWriter(stream); + + writer.Write(0.0); // X + writer.Write(0.0); // Y + writer.Write(0.0); // Z + writer.Write((double)e.Yaw); + writer.Write((double)e.Pitch); + writer.Write((double)e.Roll); + + var packet = stream.ToArray(); + _udpClient.Send(packet, packet.Length, _endpoint); + } + catch (Exception ex) + { + Log.Verbose(ex, "OpenTrackBroadcaster: Failed sending packet"); + } + } + + public void Dispose() + { + IsEnabled = false; + } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/OscSpatialBroadcaster.cs b/GalaxyBudsClient/Platform/SpatialAudio/OscSpatialBroadcaster.cs new file mode 100644 index 000000000..e767bc601 --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/OscSpatialBroadcaster.cs @@ -0,0 +1,137 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Numerics; +using System.Text; +using Serilog; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +/// +/// Broadcasts spatial head tracking data via Open Sound Control (OSC) protocol over UDP. +/// Compatible with digital audio workstations (DAWs) like Reaper, Logic Pro, and 3D spatial audio plugins (IEM, Dolby Atmos). +/// +public class OscSpatialBroadcaster : IDisposable +{ + private UdpClient? _udpClient; + private IPEndPoint? _endpoint; + private bool _isEnabled; + + public bool IsEnabled + { + get => _isEnabled; + set + { + if (_isEnabled == value) return; + _isEnabled = value; + if (_isEnabled) + { + InitSocket(); + SpatialAudioService.Instance.OrientationUpdated += OnOrientationUpdated; + } + else + { + SpatialAudioService.Instance.OrientationUpdated -= OnOrientationUpdated; + CloseSocket(); + } + } + } + + public string Host { get; set; } = "127.0.0.1"; + public int Port { get; set; } = 9000; + + private void InitSocket() + { + try + { + _endpoint = new IPEndPoint(IPAddress.Parse(Host), Port); + _udpClient = new UdpClient(); + Log.Information("OscSpatialBroadcaster: Started broadcasting to {Host}:{Port}", Host, Port); + } + catch (Exception ex) + { + Log.Error(ex, "OscSpatialBroadcaster: Failed to initialize UDP client"); + _isEnabled = false; + } + } + + private void CloseSocket() + { + try + { + _udpClient?.Close(); + _udpClient?.Dispose(); + _udpClient = null; + _endpoint = null; + } + catch (Exception ex) + { + Log.Warning(ex, "OscSpatialBroadcaster: Error while closing socket"); + } + } + + private void OnOrientationUpdated(object? sender, SpatialOrientationEventArgs e) + { + if (!_isEnabled || _udpClient == null || _endpoint == null) + return; + + try + { + // Send /spatial/ypr (yaw, pitch, roll in degrees) + var packet = CreateOscMessage("/spatial/ypr", ",fff", e.Yaw, e.Pitch, e.Roll); + _udpClient.Send(packet, packet.Length, _endpoint); + + // Send /spatial/quaternion (x, y, z, w) + var q = e.RelativeQuaternion; + var quatPacket = CreateOscMessage("/spatial/quaternion", ",ffff", q.X, q.Y, q.Z, q.W); + _udpClient.Send(quatPacket, quatPacket.Length, _endpoint); + } + catch (Exception ex) + { + Log.Verbose(ex, "OscSpatialBroadcaster: Failed sending packet"); + } + } + + private static byte[] CreateOscMessage(string address, string typeTag, params float[] values) + { + using var stream = new MemoryStream(); + using var writer = new BinaryWriter(stream); + + // Address padded to 4 bytes + WritePaddedString(writer, address); + + // Type tag padded to 4 bytes + WritePaddedString(writer, typeTag); + + // Floats in Big-Endian (network order) + foreach (var val in values) + { + var bytes = BitConverter.GetBytes(val); + if (BitConverter.IsLittleEndian) + Array.Reverse(bytes); + writer.Write(bytes); + } + + return stream.ToArray(); + } + + private static void WritePaddedString(BinaryWriter writer, string value) + { + var bytes = Encoding.ASCII.GetBytes(value); + writer.Write(bytes); + writer.Write((byte)0); // Null terminator + + var pad = 4 - ((bytes.Length + 1) % 4); + if (pad < 4) + { + for (var i = 0; i < pad; i++) + writer.Write((byte)0); + } + } + + public void Dispose() + { + IsEnabled = false; + } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioService.cs b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioService.cs new file mode 100644 index 000000000..9306eecf8 --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioService.cs @@ -0,0 +1,203 @@ +using System; +using System.Numerics; +using GalaxyBudsClient.Message; +using GalaxyBudsClient.Model.Constants; +using GalaxyBudsClient.Model.Specifications; +using GalaxyBudsClient.Platform; +using GalaxyBudsClient.Utils.Extensions; +using ReactiveUI; +using Serilog; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +public class SpatialOrientationEventArgs : EventArgs +{ + public float Yaw { get; } + public float Pitch { get; } + public float Roll { get; } + public Quaternion RelativeQuaternion { get; } + public Quaternion RawQuaternion { get; } + + public SpatialOrientationEventArgs(float yaw, float pitch, float roll, Quaternion relativeQuaternion, Quaternion rawQuaternion) + { + Yaw = yaw; + Pitch = pitch; + Roll = roll; + RelativeQuaternion = relativeQuaternion; + RawQuaternion = rawQuaternion; + } +} + +public sealed class SpatialAudioService : ReactiveObject, IDisposable +{ + private static readonly object Padlock = new(); + private static SpatialAudioService? _instance; + public static SpatialAudioService Instance + { + get + { + lock (Padlock) + { + return _instance ??= new SpatialAudioService(); + } + } + } + + private SpatialSensorManager? _sensorManager; + private Quaternion _referenceQuaternion = Quaternion.Identity; + private Quaternion _filteredQuaternion = Quaternion.Identity; + private bool _hasReference; + + public event EventHandler? OrientationUpdated; + + private bool _isActive; + public bool IsActive + { + get => _isActive; + private set => this.RaiseAndSetIfChanged(ref _isActive, value); + } + + private float _currentYaw; + public float CurrentYaw + { + get => _currentYaw; + private set => this.RaiseAndSetIfChanged(ref _currentYaw, value); + } + + private float _currentPitch; + public float CurrentPitch + { + get => _currentPitch; + private set => this.RaiseAndSetIfChanged(ref _currentPitch, value); + } + + private float _currentRoll; + public float CurrentRoll + { + get => _currentRoll; + private set => this.RaiseAndSetIfChanged(ref _currentRoll, value); + } + + public bool IsSupported => BluetoothImpl.Instance.DeviceSpec.Supports(Features.SpatialSensor) || + BluetoothImpl.Instance.DeviceSpec.Supports(Features.HeadTracking); + + private SpatialAudioService() + { + BluetoothImpl.Instance.Disconnected += OnDisconnected; + BluetoothImpl.Instance.BluetoothError += (_, _) => Stop(); + } + + public void Start() + { + if (IsActive) + return; + + if (!BluetoothImpl.Instance.IsConnected) + { + Log.Warning("SpatialAudioService: Cannot start, device not connected"); + return; + } + + try + { + Log.Information("SpatialAudioService: Starting head tracking sensor"); + _sensorManager = new SpatialSensorManager(); + _sensorManager.NewQuaternionReceived += OnNewQuaternionReceived; + _sensorManager.Attach(); + _hasReference = false; + IsActive = true; + } + catch (Exception ex) + { + Log.Error(ex, "SpatialAudioService: Failed to start sensor"); + Stop(); + } + } + + public void Stop() + { + if (!IsActive && _sensorManager == null) + return; + + Log.Information("SpatialAudioService: Stopping head tracking sensor"); + try + { + if (_sensorManager != null) + { + _sensorManager.NewQuaternionReceived -= OnNewQuaternionReceived; + _sensorManager.Detach(); + _sensorManager.Dispose(); + _sensorManager = null; + } + } + catch (Exception ex) + { + Log.Warning(ex, "SpatialAudioService: Error while stopping sensor"); + } + + IsActive = false; + CurrentYaw = 0; + CurrentPitch = 0; + CurrentRoll = 0; + _hasReference = false; + } + + public void Toggle() + { + if (IsActive) + Stop(); + else + Start(); + } + + public void Recenter() + { + if (_filteredQuaternion != Quaternion.Identity) + { + _referenceQuaternion = _filteredQuaternion; + _hasReference = true; + Log.Debug("SpatialAudioService: Recentered / Tared orientation to {Reference}", _referenceQuaternion); + } + } + + private void OnNewQuaternionReceived(object? sender, Quaternion raw) + { + if (!_hasReference) + { + _referenceQuaternion = raw; + _filteredQuaternion = raw; + _hasReference = true; + } + + // Slerp smoothing (factor 0.35 gives responsive feel with zero jitter) + _filteredQuaternion = Quaternion.Slerp(_filteredQuaternion, raw, 0.35f); + + // Compute relative rotation: q_rel = q_ref^-1 * q_current + var invRef = Quaternion.Inverse(_referenceQuaternion); + var relQuat = Quaternion.Normalize(Quaternion.Multiply(invRef, _filteredQuaternion)); + + // Convert to Euler angles (Roll, Pitch, Yaw) + var (rollRad, pitchRad, yawRad) = relQuat.ToRollPitchYaw(); + + var yawDeg = (float)(yawRad * (180.0 / Math.PI)); + var pitchDeg = (float)(pitchRad * (180.0 / Math.PI)); + var rollDeg = (float)(rollRad * (180.0 / Math.PI)); + + CurrentYaw = yawDeg; + CurrentPitch = pitchDeg; + CurrentRoll = rollDeg; + + OrientationUpdated?.Invoke(this, new SpatialOrientationEventArgs(yawDeg, pitchDeg, rollDeg, relQuat, raw)); + } + + private void OnDisconnected(object? sender, string e) + { + Stop(); + } + + public void Dispose() + { + Stop(); + BluetoothImpl.Instance.Disconnected -= OnDisconnected; + } +} diff --git a/GalaxyBudsClient/i18n/br.axaml b/GalaxyBudsClient/i18n/br.axaml index 19297a2d6..17775f1a8 100644 --- a/GalaxyBudsClient/i18n/br.axaml +++ b/GalaxyBudsClient/i18n/br.axaml @@ -1,4 +1,4 @@ - + @@ -720,4 +720,21 @@ Detalhes: Fones de ouvido renomeados Fones de ouvido renomeados com sucesso! A mudança de nome pode não ser detectada por dispositivos (incluindo este) até que você desfaça o pareamento e re-pareie os fones de ouvido. + + + Áudio 360 + Rastreamento dinâmico da cabeça e som espacial 360 + Ativar Rastreamento de Cabeça + Ativa os sensores de movimento internos para rastrear a orientação da cabeça + Recentralizar + Redefine a orientação frontal para onde você está olhando agora + Tocar Demonstração Áudio 360 + Parar Demonstração + Toca sinos binaurais interativos que giram em 3D ao redor de você conforme você move a cabeça + Transmitir via OSC (DAWs / Plugins) + Envia yaw, pitch, roll e quaternion em tempo real para localhost:9000 + Transmitir para Jogos (OpenTrack) + Envia pacotes UDP FreeTrack / OpenTrack para localhost:4242 para simuladores de voo/corrida + Rastreamento ativo + Rastreamento desativado \ No newline at end of file diff --git a/GalaxyBudsClient/i18n/en.axaml b/GalaxyBudsClient/i18n/en.axaml index aa1971b64..1f178ebef 100644 --- a/GalaxyBudsClient/i18n/en.axaml +++ b/GalaxyBudsClient/i18n/en.axaml @@ -733,4 +733,21 @@ Details: Earbuds renamed Successfully renamed your earbuds! The name change may not be detected by devices (including this one) until you un-pair and re-pair your earbuds. + + + 360 Audio + Dynamic head tracking and 360 spatial sound + Enable Head Tracking + Activates internal motion sensors to track head orientation + Recenter Heading + Resets front orientation to where you are currently looking + Play 360 Audio Demo + Stop 360 Audio Demo + Plays an interactive spatial chime that moves in 3D around you as you turn your head + Broadcast via OSC (DAWs / Plugins) + Sends real-time yaw, pitch, roll and quaternion to localhost:9000 + Broadcast to Games (OpenTrack) + Sends FreeTrack / OpenTrack UDP packets to localhost:4242 for flight/driving sims + Tracking active + Tracking stopped \ No newline at end of file diff --git a/GalaxyBudsClient/i18n/pt.axaml b/GalaxyBudsClient/i18n/pt.axaml index 049cc56e9..36c9c0310 100644 --- a/GalaxyBudsClient/i18n/pt.axaml +++ b/GalaxyBudsClient/i18n/pt.axaml @@ -1,4 +1,4 @@ - + @@ -720,4 +720,21 @@ Detalhes: Fones de ouvido renomeados Fones de ouvido renomeados com sucesso! A mudança de nome pode não ser detectada por dispositivos (incluindo este) até que você desfaça o pareamento e re-pareie os fones de ouvido. + + + Áudio 360 + Rastreamento dinâmico da cabeça e som espacial 360 + Ativar Rastreamento de Cabeça + Ativa os sensores de movimento internos para rastrear a orientação da cabeça + Recentralizar + Redefine a orientação frontal para onde você está olhando agora + Tocar Demonstração Áudio 360 + Parar Demonstração + Toca sinos binaurais interativos que giram em 3D ao redor de você conforme você move a cabeça + Transmitir via OSC (DAWs / Plugins) + Envia yaw, pitch, roll e quaternion em tempo real para localhost:9000 + Transmitir para Jogos (OpenTrack) + Envia pacotes UDP FreeTrack / OpenTrack para localhost:4242 para simuladores de voo/corrida + Rastreamento ativo + Rastreamento desativado \ No newline at end of file From 7c0c62ba3e09c50505383fef1d20668b86700d5b Mon Sep 17 00:00:00 2001 From: Juan Pablo Farias Date: Fri, 4 Sep 2026 16:36:21 -0300 Subject: [PATCH 3/6] feat: implement macOS audio output sink and update spatial orientation calculation logic --- GalaxyBudsClient.Tests/SpatialAudioTests.cs | 100 +++++- .../Interface/Pages/SpatialAudioPage.axaml | 19 ++ .../Pages/SpatialAudioPageViewModel.cs | 60 ++-- .../SpatialAudio/ISpatialAudioSink.cs | 11 + .../SpatialAudio/MacOsSpatialAudioSink.cs | 182 +++++++++++ .../SpatialAudio/SpatialAudioDspEngine.cs | 303 ++++++++++++++++++ .../SpatialAudio/SpatialAudioService.cs | 12 +- .../SpatialAudio/SpatialAudioSinkFactory.cs | 43 +++ .../SpatialAudio/SpatialMediaPlayer.cs | 225 +++++++++++++ .../Utils/Extensions/MathExtensions.cs | 3 +- GalaxyBudsClient/i18n/br.axaml | 4 + GalaxyBudsClient/i18n/en.axaml | 4 + GalaxyBudsClient/i18n/pt.axaml | 4 + 13 files changed, 946 insertions(+), 24 deletions(-) create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/ISpatialAudioSink.cs create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/MacOsSpatialAudioSink.cs create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioSinkFactory.cs create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/SpatialMediaPlayer.cs diff --git a/GalaxyBudsClient.Tests/SpatialAudioTests.cs b/GalaxyBudsClient.Tests/SpatialAudioTests.cs index 787b7a1f7..13ccab955 100644 --- a/GalaxyBudsClient.Tests/SpatialAudioTests.cs +++ b/GalaxyBudsClient.Tests/SpatialAudioTests.cs @@ -33,7 +33,7 @@ public void RelativeOrientation_IdenticalQuaternions_ResultsInZeroDelta() var currentQuat = refQuat; var invRef = Quaternion.Inverse(refQuat); - var rel = Quaternion.Normalize(Quaternion.Multiply(invRef, currentQuat)); + var rel = Quaternion.Normalize(Quaternion.Multiply(currentQuat, invRef)); var (roll, pitch, yaw) = rel.ToRollPitchYaw(); ((double)yaw).Should().BeApproximately(0.0, 0.001); @@ -49,7 +49,7 @@ public void RelativeOrientation_YawOffset_CalculatesExactDegrees() var currentQuat = Quaternion.CreateFromAxisAngle(Vector3.UnitZ, (float)(Math.PI / 2.0)); var invRef = Quaternion.Inverse(refQuat); - var rel = Quaternion.Normalize(Quaternion.Multiply(invRef, currentQuat)); + var rel = Quaternion.Normalize(Quaternion.Multiply(currentQuat, invRef)); var (_, _, yawRad) = rel.ToRollPitchYaw(); var yawDeg = (double)yawRad * (180.0 / Math.PI); @@ -57,6 +57,30 @@ public void RelativeOrientation_YawOffset_CalculatesExactDegrees() yawDeg.Should().BeApproximately(90.0, 0.1); } + [Test] + public void RelativeOrientation_TiltedEarbudInEar_DecouplesPureHorizontalYaw() + { + // Real-world scenario: Earbud sits tilted ~45° in ear canal + var earbudTilt = Quaternion.CreateFromYawPitchRoll(0.8f, -0.4f, 0.5f); + var refQuat = earbudTilt; + + // User turns head horizontally in the room by +35 degrees around World Z + var turnAngleRad = (float)(35.0 * Math.PI / 180.0); + var worldTurn = Quaternion.CreateFromAxisAngle(Vector3.UnitZ, turnAngleRad); + var currentQuat = Quaternion.Multiply(worldTurn, refQuat); + + // World frame relative rotation + var invRef = Quaternion.Inverse(refQuat); + var rel = Quaternion.Normalize(Quaternion.Multiply(currentQuat, invRef)); + + var (roll, pitch, yawRad) = rel.ToRollPitchYaw(); + var yawDeg = (double)yawRad * (180.0 / Math.PI); + + yawDeg.Should().BeApproximately(35.0, 0.01); + ((double)roll).Should().BeApproximately(0.0, 0.01); + ((double)pitch).Should().BeApproximately(0.0, 0.01); + } + [Test] public void OpenTrackBroadcaster_GeneratesValid48BytePacket() { @@ -102,4 +126,76 @@ public void OscMessage_AddressAndTypeTag_PaddedToFourBytes() var tagPaddedLen = ((tagBytes.Length + 4) / 4) * 4; tagPaddedLen.Should().Be(8); } + + [Test] + public void DspEngine_CenterOrientation_ProducesBalancedSymmetricEnergy() + { + var engine = new SpatialAudioDspEngine(48000); + engine.SetOrientation(0, 0, 0); + + var samples = 4800; // 100ms + var input = new float[samples * 2]; + var output = new float[samples * 2]; + + // Fill with stereo tone + for (var i = 0; i < samples; i++) + { + var val = MathF.Sin(2.0f * MathF.PI * 440.0f * (i / 48000.0f)); + input[i * 2] = val; + input[i * 2 + 1] = val; + } + + // Warm up and process + engine.Process(input, output); + + // Sum energy on left and right ears + float energyL = 0f; + float energyR = 0f; + for (var i = samples / 2; i < samples; i++) + { + energyL += output[i * 2] * output[i * 2]; + energyR += output[i * 2 + 1] * output[i * 2 + 1]; + } + + // At center, left and right energy must be symmetrical + ((double)MathF.Abs(energyL - energyR) / energyL).Should().BeLessThan(0.05); + } + + [Test] + public void DspEngine_TurnHeadLeft_ShiftsSoundEnergyToRightEar() + { + var engine = new SpatialAudioDspEngine(48000); + // Turn head 60 degrees to the left (-60 Yaw) + // Speakers in front of monitor are now to the right of the head + engine.SetOrientation(-60, 0, 0); + + var samples = 4800; + var input = new float[samples * 2]; + var output = new float[samples * 2]; + + for (var i = 0; i < samples; i++) + { + var val = MathF.Sin(2.0f * MathF.PI * 1000.0f * (i / 48000.0f)); + input[i * 2] = val; + input[i * 2 + 1] = val; + } + + // Process warm up + for (var pass = 0; pass < 5; pass++) + { + engine.Process(input, output); + } + + float energyL = 0f; + float energyR = 0f; + for (var i = 0; i < samples; i++) + { + energyL += output[i * 2] * output[i * 2]; + energyR += output[i * 2 + 1] * output[i * 2 + 1]; + } + + // Right ear must receive significantly more energy than left ear (head shadow + ILD) + energyR.Should().BeGreaterThan(energyL * 1.5f); + } } + diff --git a/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml index 91b16a94c..3a5237f76 100644 --- a/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml +++ b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml @@ -136,6 +136,25 @@ IsChecked="{Binding IsTrackingEnabled}" /> + + + + + + + + { + IsDemoPlaying = SpatialMediaPlayer.Instance.IsPlaying; + }); + } + } + private void OnServicePropertyChanged(object? sender, PropertyChangedEventArgs e) { if (e.PropertyName == nameof(SpatialAudioService.IsActive)) @@ -72,6 +87,14 @@ private void OnSelfPropertyChanged(object? sender, PropertyChangedEventArgs e) case nameof(IsOpenTrackBroadcasting): _openTrackBroadcaster.IsEnabled = IsOpenTrackBroadcasting; break; + + case nameof(SpeakerAngle): + SpatialMediaPlayer.Instance.VirtualSpeakerAngle = (float)SpeakerAngle; + break; + + case nameof(AmbiencePercent): + SpatialMediaPlayer.Instance.AmbienceAmount = (float)(AmbiencePercent / 100.0); + break; } } @@ -85,24 +108,16 @@ private void OnOrientationUpdated(object? sender, SpatialOrientationEventArgs e) }, DispatcherPriority.Render); } - private void OnPlaybackStateChanged(object? sender, bool isPlaying) - { - Dispatcher.UIThread.Post(() => - { - IsDemoPlaying = isPlaying; - }); - } - public void Recenter() { SpatialAudioService.Instance.Recenter(); } - public async void ToggleDemo() + public void ToggleDemo() { - if (IsDemoPlaying) + if (SpatialMediaPlayer.Instance.IsPlaying) { - _demoPlayer.Stop(); + SpatialMediaPlayer.Instance.Stop(); } else { @@ -110,7 +125,7 @@ public async void ToggleDemo() { IsTrackingEnabled = true; } - await _demoPlayer.StartAsync(); + SpatialMediaPlayer.Instance.Play(); } } @@ -119,12 +134,21 @@ private void UpdateStatusText() StatusText = IsTrackingEnabled ? Strings.SpatialTrackingActive : Strings.SpatialTrackingInactive; } + public override void OnNavigatedTo() + { + base.OnNavigatedTo(); + if (SpatialAudioService.Instance.IsSupported && !SpatialAudioService.Instance.IsActive) + { + IsTrackingEnabled = true; + } + } + public override void OnNavigatedFrom() { base.OnNavigatedFrom(); - if (IsDemoPlaying) + if (SpatialMediaPlayer.Instance.IsPlaying) { - _demoPlayer.Stop(); + SpatialMediaPlayer.Instance.Stop(); } } @@ -138,8 +162,8 @@ public void Dispose() { SpatialAudioService.Instance.OrientationUpdated -= OnOrientationUpdated; SpatialAudioService.Instance.PropertyChanged -= OnServicePropertyChanged; - _demoPlayer.PlaybackStateChanged -= OnPlaybackStateChanged; - _demoPlayer.Dispose(); + SpatialMediaPlayer.Instance.PropertyChanged -= OnMediaPlayerPropertyChanged; + SpatialMediaPlayer.Instance.Stop(); _oscBroadcaster.Dispose(); _openTrackBroadcaster.Dispose(); GC.SuppressFinalize(this); diff --git a/GalaxyBudsClient/Platform/SpatialAudio/ISpatialAudioSink.cs b/GalaxyBudsClient/Platform/SpatialAudio/ISpatialAudioSink.cs new file mode 100644 index 000000000..0c64d5cf4 --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/ISpatialAudioSink.cs @@ -0,0 +1,11 @@ +using System; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +public interface ISpatialAudioSink : IDisposable +{ + void Start(Func, int> readStereoSamplesCallback); + void Stop(); + bool IsRunning { get; } + int SampleRate { get; } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/MacOsSpatialAudioSink.cs b/GalaxyBudsClient/Platform/SpatialAudio/MacOsSpatialAudioSink.cs new file mode 100644 index 000000000..4425a7c1e --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/MacOsSpatialAudioSink.cs @@ -0,0 +1,182 @@ +using System; +using System.Runtime.InteropServices; +using Serilog; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +public sealed unsafe class MacOsSpatialAudioSink : ISpatialAudioSink +{ + private const string AudioToolboxLib = "/System/Library/Frameworks/AudioToolbox.framework/AudioToolbox"; + + [StructLayout(LayoutKind.Sequential)] + private struct AudioStreamBasicDescription + { + public double mSampleRate; + public uint mFormatID; + public uint mFormatFlags; + public uint mBytesPerPacket; + public uint mFramesPerPacket; + public uint mBytesPerFrame; + public uint mChannelsPerFrame; + public uint mBitsPerChannel; + public uint mReserved; + } + + [StructLayout(LayoutKind.Sequential)] + private struct AudioQueueBuffer + { + public uint mAudioDataBytesCapacity; + public void* mAudioData; + public uint mAudioDataByteSize; + public void* mUserData; + public uint mPacketDescriptionCapacity; + public void* mPacketDescriptions; + public uint mPacketDescriptionCount; + } + + private delegate void AudioQueueOutputCallback(IntPtr userData, IntPtr aq, IntPtr buffer); + + [DllImport(AudioToolboxLib)] + private static extern int AudioQueueNewOutput( + ref AudioStreamBasicDescription inFormat, + AudioQueueOutputCallback inCallbackProc, + IntPtr inUserData, + IntPtr inCallbackRunLoop, + IntPtr inCallbackRunLoopMode, + uint inFlags, + out IntPtr outAq); + + [DllImport(AudioToolboxLib)] + private static extern int AudioQueueAllocateBuffer(IntPtr inAq, uint inBufferByteSize, out IntPtr outBuffer); + + [DllImport(AudioToolboxLib)] + private static extern int AudioQueueEnqueueBuffer(IntPtr inAq, IntPtr inBuffer, uint inNumPacketDescs, IntPtr inPacketDescs); + + [DllImport(AudioToolboxLib)] + private static extern int AudioQueueStart(IntPtr inAq, IntPtr inStartTime); + + [DllImport(AudioToolboxLib)] + private static extern int AudioQueueStop(IntPtr inAq, bool inImmediate); + + [DllImport(AudioToolboxLib)] + private static extern int AudioQueueDispose(IntPtr inAq, bool inImmediate); + + private const int BufferCount = 3; + private const int BufferFrames = 1024; // ~21ms at 48kHz + private const int ChannelCount = 2; + + private IntPtr _audioQueue; + private readonly IntPtr[] _buffers = new IntPtr[BufferCount]; + private AudioQueueOutputCallback? _callback; + private Func, int>? _readSamplesCallback; + private bool _isRunning; + private readonly object _lock = new(); + + public bool IsRunning => _isRunning; + public int SampleRate { get; } + + public MacOsSpatialAudioSink(int sampleRate = 48000) + { + SampleRate = sampleRate; + } + + public void Start(Func, int> readStereoSamplesCallback) + { + lock (_lock) + { + if (_isRunning) return; + _readSamplesCallback = readStereoSamplesCallback; + + var desc = new AudioStreamBasicDescription + { + mSampleRate = SampleRate, + mFormatID = 0x6c70636d, // 'lpcm' + mFormatFlags = (1 << 0) | (1 << 3), // kAudioFormatFlagIsFloat (1) | kAudioFormatFlagIsPacked (8) = 0x9 + mBytesPerPacket = 8, + mFramesPerPacket = 1, + mBytesPerFrame = 8, + mChannelsPerFrame = ChannelCount, + mBitsPerChannel = 32, + mReserved = 0 + }; + + _callback = OnBufferComplete; + var err = AudioQueueNewOutput(ref desc, _callback, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, 0, out _audioQueue); + if (err != 0) + { + Log.Error("MacOsSpatialAudioSink: AudioQueueNewOutput failed with error {Error}", err); + return; + } + + var bufferBytes = (uint)(BufferFrames * ChannelCount * sizeof(float)); + for (var i = 0; i < BufferCount; i++) + { + var allocErr = AudioQueueAllocateBuffer(_audioQueue, bufferBytes, out _buffers[i]); + if (allocErr != 0) + { + Log.Error("MacOsSpatialAudioSink: AudioQueueAllocateBuffer failed with {Error}", allocErr); + } + FillAndEnqueueBuffer(_buffers[i]); + } + + var startErr = AudioQueueStart(_audioQueue, IntPtr.Zero); + if (startErr != 0) + { + Log.Error("MacOsSpatialAudioSink: AudioQueueStart failed with {Error}", startErr); + } + _isRunning = true; + Log.Information("MacOsSpatialAudioSink: Started low-latency audio queue ({SampleRate}Hz)", SampleRate); + } + } + + private void FillAndEnqueueBuffer(IntPtr bufferPtr) + { + var buffer = (AudioQueueBuffer*)bufferPtr; + var maxSamples = (int)(buffer->mAudioDataBytesCapacity / sizeof(float)); + var floatSpan = new Span(buffer->mAudioData, maxSamples); + + var samplesFilled = _readSamplesCallback?.Invoke(floatSpan) ?? 0; + if (samplesFilled <= 0) + { + floatSpan.Clear(); + samplesFilled = maxSamples; + } + + buffer->mAudioDataByteSize = (uint)(samplesFilled * sizeof(float)); + var enqErr = AudioQueueEnqueueBuffer(_audioQueue, bufferPtr, 0, IntPtr.Zero); + if (enqErr != 0) + { + Log.Warning("MacOsSpatialAudioSink: AudioQueueEnqueueBuffer failed with {Error}", enqErr); + } + } + + private void OnBufferComplete(IntPtr userData, IntPtr aq, IntPtr bufferPtr) + { + if (!_isRunning) return; + FillAndEnqueueBuffer(bufferPtr); + } + + public void Stop() + { + lock (_lock) + { + if (!_isRunning) return; + _isRunning = false; + + if (_audioQueue != IntPtr.Zero) + { + AudioQueueStop(_audioQueue, true); + AudioQueueDispose(_audioQueue, true); + _audioQueue = IntPtr.Zero; + } + + _readSamplesCallback = null; + Log.Information("MacOsSpatialAudioSink: Stopped"); + } + } + + public void Dispose() + { + Stop(); + } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs new file mode 100644 index 000000000..74cf1282a --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs @@ -0,0 +1,303 @@ +using System; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +/// +/// High-performance, zero-allocation real-time DSP engine for 360 binaural spatial audio. +/// Takes standard stereo audio (e.g. system sound, YouTube, Spotify, movies) and renders +/// virtual speakers anchored in 3D space in front of the listener using Woodworth ITD, +/// frequency-dependent ILD head shadow biquad filters, and subtle early room reflections. +/// +public sealed class SpatialAudioDspEngine +{ + private readonly int _sampleRate; + private readonly float _invSampleRate; + + // Head physical model parameters + private const float HeadRadiusMeters = 0.0875f; // ~8.75 cm + private const float SpeedOfSound = 343.0f; // m/s + private readonly float _maxDelaySeconds; + private readonly float _maxDelaySamples; + + // Fractional delay lines for each ear: [0] = Left Ear, [1] = Right Ear + // Circular buffer length of 1024 is plenty for ~32 samples max delay + early reflections + private const int DelayBufferSize = 1024; + private const int DelayMask = DelayBufferSize - 1; + private readonly float[] _delayBufferLeftIn = new float[DelayBufferSize]; + private readonly float[] _delayBufferRightIn = new float[DelayBufferSize]; + private int _writeIndex; + + // Current smoothed orientation (in radians) + private float _targetYaw; + private float _targetPitch; + private float _targetRoll; + + private float _smoothedYaw; + private float _smoothedPitch; + private float _smoothedRoll; + + // Filter states for Head Shadow (Biquad low-shelf / low-pass for each channel and ear) + // Left input -> Left Ear, Left input -> Right Ear, Right input -> Left Ear, Right input -> Right Ear + private struct BiquadState + { + public float X1, X2; + public float Y1, Y2; + + public void Reset() + { + X1 = X2 = Y1 = Y2 = 0f; + } + } + + private BiquadState _filterLL; + private BiquadState _filterLR; + private BiquadState _filterRL; + private BiquadState _filterRR; + + // Configuration + public float SpeakerAzimuthDeg { get; set; } = 30.0f; // Virtual stereo speakers at ±30° + public float AmbienceAmount { get; set; } = 0.12f; // Subtle cross-reflection to externalize sound + + public SpatialAudioDspEngine(int sampleRate = 48000) + { + _sampleRate = Math.Max(22050, Math.Min(192000, sampleRate)); + _invSampleRate = 1.0f / _sampleRate; + _maxDelaySeconds = (HeadRadiusMeters / SpeedOfSound) * ((MathF.PI / 2.0f) + 1.0f); + _maxDelaySamples = _maxDelaySeconds * _sampleRate; + } + + /// + /// Update the listener's head orientation in degrees. + /// + public void SetOrientation(float yawDegrees, float pitchDegrees, float rollDegrees) + { + _targetYaw = yawDegrees * (MathF.PI / 180.0f); + _targetPitch = pitchDegrees * (MathF.PI / 180.0f); + _targetRoll = rollDegrees * (MathF.PI / 180.0f); + } + + /// + /// Process interleaved 32-bit floating point stereo PCM audio frames. + /// + public void Process(ReadOnlySpan input, Span output) + { + var frameCount = Math.Min(input.Length / 2, output.Length / 2); + if (frameCount <= 0) return; + + // Smoothing factor for parameter transitions (per sample) + const float smoothAlpha = 0.005f; + + var speakerAngleRad = SpeakerAzimuthDeg * (MathF.PI / 180.0f); + + for (var i = 0; i < frameCount; i++) + { + var inIdx = i * 2; + var inL = input[inIdx]; + var inR = input[inIdx + 1]; + + // Smooth orientation + _smoothedYaw += (_targetYaw - _smoothedYaw) * smoothAlpha; + _smoothedPitch += (_targetPitch - _smoothedPitch) * smoothAlpha; + _smoothedRoll += (_targetRoll - _smoothedRoll) * smoothAlpha; + + // Write input samples to circular delay lines + _delayBufferLeftIn[_writeIndex] = inL; + _delayBufferRightIn[_writeIndex] = inR; + + // Compute virtual speaker relative angles + // Left virtual speaker is at -speakerAngleRad in world space + // Relative azimuth to head: relAngle = worldAngle - yaw + var relAngleL = -speakerAngleRad - _smoothedYaw; + var relAngleR = speakerAngleRad - _smoothedYaw; + + // Wrap relative angles to [-PI, PI] + relAngleL = NormalizeAngle(relAngleL); + relAngleR = NormalizeAngle(relAngleR); + + // Calculate ITD (delays in samples) and ILD (gain & filter cutoff) for: + // 1. Left virtual speaker to Left and Right ears + CalculateEarResponse(relAngleL, out var delayL_to_L, out var gainL_to_L, out var cutoffL_to_L, + out var delayL_to_R, out var gainL_to_R, out var cutoffL_to_R); + + // 2. Right virtual speaker to Left and Right ears + CalculateEarResponse(relAngleR, out var delayR_to_L, out var gainR_to_L, out var cutoffR_to_L, + out var delayR_to_R, out var gainR_to_R, out var cutoffR_to_R); + + // Read delayed samples with linear fractional interpolation + var sampleLL = ReadFractionalDelay(_delayBufferLeftIn, _writeIndex, delayL_to_L); + var sampleLR = ReadFractionalDelay(_delayBufferLeftIn, _writeIndex, delayL_to_R); + var sampleRL = ReadFractionalDelay(_delayBufferRightIn, _writeIndex, delayR_to_L); + var sampleRR = ReadFractionalDelay(_delayBufferRightIn, _writeIndex, delayR_to_R); + + // Apply ILD head shadow filters (frequency-dependent attenuation) + sampleLL = ApplyHeadShadowFilter(sampleLL, cutoffL_to_L, ref _filterLL) * gainL_to_L; + sampleLR = ApplyHeadShadowFilter(sampleLR, cutoffL_to_R, ref _filterLR) * gainL_to_R; + sampleRL = ApplyHeadShadowFilter(sampleRL, cutoffR_to_L, ref _filterRL) * gainR_to_L; + sampleRR = ApplyHeadShadowFilter(sampleRR, cutoffR_to_R, ref _filterRR) * gainR_to_R; + + // Subtle early reflections to simulate studio room acoustic cues + var earlyRefL = ReadFractionalDelay(_delayBufferLeftIn, _writeIndex, delayL_to_L + 120.0f) * AmbienceAmount; + var earlyRefR = ReadFractionalDelay(_delayBufferRightIn, _writeIndex, delayR_to_R + 140.0f) * AmbienceAmount; + + // Sum outputs for Left Ear and Right Ear + var outL = sampleLL + sampleRL + (earlyRefR * 0.5f); + var outR = sampleLR + sampleRR + (earlyRefL * 0.5f); + + // Soft clipper to prevent any digital distortion + output[inIdx] = SoftClip(outL); + output[inIdx + 1] = SoftClip(outR); + + _writeIndex = (_writeIndex + 1) & DelayMask; + } + } + + /// + /// Process interleaved 16-bit signed PCM audio bytes. + /// + public void Process16Bit(ReadOnlySpan inputBytes, Span outputBytes) + { + var sampleCount = Math.Min(inputBytes.Length / 2, outputBytes.Length / 2); + var frameCount = sampleCount / 2; + if (frameCount <= 0) return; + + // Process in chunks of up to 256 frames on stack to avoid heap allocation + Span floatIn = stackalloc float[512]; + Span floatOut = stackalloc float[512]; + + var processedFrames = 0; + while (processedFrames < frameCount) + { + var chunkFrames = Math.Min(256, frameCount - processedFrames); + var chunkSamples = chunkFrames * 2; + + var inByteOffset = processedFrames * 4; + for (var s = 0; s < chunkSamples; s++) + { + var byteIdx = inByteOffset + (s * 2); + var sampleInt16 = (short)(inputBytes[byteIdx] | (inputBytes[byteIdx + 1] << 8)); + floatIn[s] = sampleInt16 / 32768.0f; + } + + Process(floatIn[..chunkSamples], floatOut[..chunkSamples]); + + var outByteOffset = processedFrames * 4; + for (var s = 0; s < chunkSamples; s++) + { + var val = (int)(floatOut[s] * 32767.0f); + val = Math.Clamp(val, -32768, 32767); + var byteIdx = outByteOffset + (s * 2); + outputBytes[byteIdx] = (byte)(val & 0xFF); + outputBytes[byteIdx + 1] = (byte)((val >> 8) & 0xFF); + } + + processedFrames += chunkFrames; + } + } + + private void CalculateEarResponse(float relAngleRad, + out float delayL, out float gainL, out float cutoffL, + out float delayR, out float gainR, out float cutoffR) + { + // Woodworth spherical head model for Left Ear (located at -PI/2) and Right Ear (+PI/2) + // Angle to Left Ear: thetaL = relAngle + PI/2 + // Angle to Right Ear: thetaR = relAngle - PI/2 + var thetaL = NormalizeAngle(relAngleRad + (MathF.PI / 2.0f)); + var thetaR = NormalizeAngle(relAngleRad - (MathF.PI / 2.0f)); + + // Interaural Time Difference (ITD) delay in seconds + var delaySecL = WoodworthDelay(thetaL); + var delaySecR = WoodworthDelay(thetaR); + + delayL = delaySecL * _sampleRate; + delayR = delaySecR * _sampleRate; + + // Interaural Level Difference (ILD) + // Standard pan: -1 when source is to listener's left, +1 when source is to listener's right + var pan = Math.Clamp(MathF.Sin(relAngleRad), -1.0f, 1.0f); + + // Constant power panning law + // pan = -1 (hard left): gainL = 1.0, gainR = 0.0 + // pan = 0 (center): gainL = 0.707, gainR = 0.707 + // pan = +1 (hard right):gainL = 0.0, gainR = 1.0 + var panAngle = (pan + 1.0f) * (MathF.PI / 4.0f); + gainL = MathF.Cos(panAngle); + gainR = MathF.Sin(panAngle); + + // Head shadow cutoff frequency: + // Ipsilateral ear gets direct high frequencies (cutoff up to 20kHz) + // Contralateral ear is shadowed by head (cutoff down to ~1.8kHz) + // Left ear is shadowed only when sound is to listener's right (pan > 0) + // Right ear is shadowed only when sound is to listener's left (pan < 0) + var shadowDepthL = Math.Clamp(pan, 0f, 1f); + var shadowDepthR = Math.Clamp(-pan, 0f, 1f); + + cutoffL = MathF.Exp(MathF.Log(1800.0f) * shadowDepthL + MathF.Log(20000.0f) * (1.0f - shadowDepthL)); + cutoffR = MathF.Exp(MathF.Log(1800.0f) * shadowDepthR + MathF.Log(20000.0f) * (1.0f - shadowDepthR)); + } + + private static float WoodworthDelay(float theta) + { + var absTheta = MathF.Abs(theta); + float delay; + if (absTheta <= MathF.PI / 2.0f) + { + delay = (HeadRadiusMeters / SpeedOfSound) * (1.0f - MathF.Cos(absTheta)); + } + else + { + delay = (HeadRadiusMeters / SpeedOfSound) * (1.0f + (absTheta - (MathF.PI / 2.0f))); + } + return Math.Max(0.0f, delay); + } + + private static float ReadFractionalDelay(float[] buffer, int writeIdx, float delaySamples) + { + var readPos = writeIdx - delaySamples; + while (readPos < 0) readPos += DelayBufferSize; + + var index0 = (int)readPos; + var frac = readPos - index0; + var index1 = (index0 + 1) & DelayMask; + + index0 &= DelayMask; + + // Linear interpolation + return (buffer[index0] * (1.0f - frac)) + (buffer[index1] * frac); + } + + private float ApplyHeadShadowFilter(float input, float cutoffHz, ref BiquadState state) + { + // 1st order low-pass IIR filter: y[n] = (1 - alpha) * x[n] + alpha * y[n-1] + var w = 2.0f * MathF.PI * cutoffHz * _invSampleRate; + var alpha = Math.Clamp(MathF.Exp(-w), 0.0f, 0.98f); + + var output = ((1.0f - alpha) * input) + (alpha * state.Y1); + state.Y1 = output; + return output; + } + + private static float SoftClip(float x) + { + if (x > 1.0f) return 1.0f - MathF.Exp(-x); + if (x < -1.0f) return -1.0f + MathF.Exp(x); + return x; + } + + private static float NormalizeAngle(float angle) + { + while (angle > MathF.PI) angle -= 2.0f * MathF.PI; + while (angle < -MathF.PI) angle += 2.0f * MathF.PI; + return angle; + } + + public void Reset() + { + Array.Clear(_delayBufferLeftIn); + Array.Clear(_delayBufferRightIn); + _filterLL.Reset(); + _filterLR.Reset(); + _filterRL.Reset(); + _filterRR.Reset(); + _writeIndex = 0; + } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioService.cs b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioService.cs index 9306eecf8..699924d09 100644 --- a/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioService.cs +++ b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioService.cs @@ -156,6 +156,10 @@ public void Recenter() { _referenceQuaternion = _filteredQuaternion; _hasReference = true; + CurrentYaw = 0; + CurrentPitch = 0; + CurrentRoll = 0; + OrientationUpdated?.Invoke(this, new SpatialOrientationEventArgs(0, 0, 0, Quaternion.Identity, _filteredQuaternion)); Log.Debug("SpatialAudioService: Recentered / Tared orientation to {Reference}", _referenceQuaternion); } } @@ -172,14 +176,16 @@ private void OnNewQuaternionReceived(object? sender, Quaternion raw) // Slerp smoothing (factor 0.35 gives responsive feel with zero jitter) _filteredQuaternion = Quaternion.Slerp(_filteredQuaternion, raw, 0.35f); - // Compute relative rotation: q_rel = q_ref^-1 * q_current + // Compute relative rotation in world frame: R_world = q_current * q_reference^-1 + // This decouples head rotations (around vertical gravity axis) cleanly from earbud placement var invRef = Quaternion.Inverse(_referenceQuaternion); - var relQuat = Quaternion.Normalize(Quaternion.Multiply(invRef, _filteredQuaternion)); + var relQuat = Quaternion.Normalize(Quaternion.Multiply(_filteredQuaternion, invRef)); // Convert to Euler angles (Roll, Pitch, Yaw) var (rollRad, pitchRad, yawRad) = relQuat.ToRollPitchYaw(); - var yawDeg = (float)(yawRad * (180.0 / Math.PI)); + // Yaw: positive when head turns RIGHT, negative when head turns LEFT (standard OpenTrack/aviation convention) + var yawDeg = -(float)(yawRad * (180.0 / Math.PI)); var pitchDeg = (float)(pitchRad * (180.0 / Math.PI)); var rollDeg = (float)(rollRad * (180.0 / Math.PI)); diff --git a/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioSinkFactory.cs b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioSinkFactory.cs new file mode 100644 index 000000000..44da6d72f --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioSinkFactory.cs @@ -0,0 +1,43 @@ +using System; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +public class NullSpatialAudioSink : ISpatialAudioSink +{ + public bool IsRunning { get; private set; } + public int SampleRate { get; } + + public NullSpatialAudioSink(int sampleRate = 48000) + { + SampleRate = sampleRate; + } + + public void Start(Func, int> readStereoSamplesCallback) + { + IsRunning = true; + } + + public void Stop() + { + IsRunning = false; + } + + public void Dispose() + { + Stop(); + } +} + +public static class SpatialAudioSinkFactory +{ + public static ISpatialAudioSink Create(int sampleRate = 48000) + { + if (OperatingSystem.IsMacOS()) + { + return new MacOsSpatialAudioSink(sampleRate); + } + + // Windows and Linux fallbacks + return new NullSpatialAudioSink(sampleRate); + } +} diff --git a/GalaxyBudsClient/Platform/SpatialAudio/SpatialMediaPlayer.cs b/GalaxyBudsClient/Platform/SpatialAudio/SpatialMediaPlayer.cs new file mode 100644 index 000000000..48b1ca7ca --- /dev/null +++ b/GalaxyBudsClient/Platform/SpatialAudio/SpatialMediaPlayer.cs @@ -0,0 +1,225 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using ReactiveUI; +using Serilog; + +namespace GalaxyBudsClient.Platform.SpatialAudio; + +/// +/// Real-time 360 spatial audio player and stream engine. +/// Continuously generates and/or streams stereo audio through the SpatialAudioDspEngine +/// rotated in real time by head orientation from the Galaxy Buds 2 Pro motion sensors, +/// outputting low-latency binaural 3D sound to the active audio device. +/// +public sealed class SpatialMediaPlayer : ReactiveObject, IDisposable +{ + private static readonly object Padlock = new(); + private static SpatialMediaPlayer? _instance; + public static SpatialMediaPlayer Instance + { + get + { + lock (Padlock) + { + return _instance ??= new SpatialMediaPlayer(); + } + } + } + + private const int SampleRate = 44100; + private readonly SpatialAudioDspEngine _dspEngine = new(SampleRate); + private ISpatialAudioSink? _audioSink; + + private bool _isPlaying; + public bool IsPlaying + { + get => _isPlaying; + private set => this.RaiseAndSetIfChanged(ref _isPlaying, value); + } + + public float VirtualSpeakerAngle + { + get => _dspEngine.SpeakerAzimuthDeg; + set + { + _dspEngine.SpeakerAzimuthDeg = Math.Clamp(value, 15f, 75f); + this.RaisePropertyChanged(); + } + } + + public float AmbienceAmount + { + get => _dspEngine.AmbienceAmount; + set + { + _dspEngine.AmbienceAmount = Math.Clamp(value, 0f, 0.4f); + this.RaisePropertyChanged(); + } + } + + // Generator state for harmonic 360 music/ambience demo + private double _sampleTime; + private float[] _rawBuffer = new float[16384]; + + private SpatialMediaPlayer() + { + SpatialAudioService.Instance.OrientationUpdated += OnOrientationUpdated; + } + + private void OnOrientationUpdated(object? sender, SpatialOrientationEventArgs e) + { + _dspEngine.SetOrientation(e.Yaw, e.Pitch, e.Roll); + } + + public void Play() + { + if (IsPlaying) return; + + try + { + Log.Information("SpatialMediaPlayer: Starting continuous 360 audio stream"); + _dspEngine.Reset(); + _dspEngine.SetOrientation(SpatialAudioService.Instance.CurrentYaw, + SpatialAudioService.Instance.CurrentPitch, + SpatialAudioService.Instance.CurrentRoll); + _audioSink = SpatialAudioSinkFactory.Create(SampleRate); + _audioSink.Start(OnProvideAudioSamples); + IsPlaying = true; + } + catch (Exception ex) + { + Log.Error(ex, "SpatialMediaPlayer: Failed to start audio playback"); + Stop(); + } + } + + public void Stop() + { + if (!IsPlaying && _audioSink == null) return; + + Log.Information("SpatialMediaPlayer: Stopping 360 audio stream"); + try + { + _audioSink?.Stop(); + _audioSink?.Dispose(); + _audioSink = null; + } + catch (Exception ex) + { + Log.Warning(ex, "SpatialMediaPlayer: Error stopping audio sink"); + } + finally + { + IsPlaying = false; + } + } + + public void Toggle() + { + if (IsPlaying) + Stop(); + else + Play(); + } + + private int OnProvideAudioSamples(Span outputBuffer) + { + var sampleCount = outputBuffer.Length; + if (_rawBuffer.Length < sampleCount) + { + _rawBuffer = new float[sampleCount * 2]; + } + + var frameCount = sampleCount / 2; + var rawSpan = _rawBuffer.AsSpan(0, sampleCount); + + // Synthesize a rich, soothing stereo ambient soundscape (anchored at physical screen): + // Chord progression: Cmaj9 -> Am9 -> Fmaj7 -> Gsus4 + var chords = new (double c1, double c2, double c3, double c4)[] + { + (261.63, 329.63, 392.00, 493.88), // Cmaj9 + (220.00, 261.63, 329.63, 392.00), // Am9 + (174.61, 220.00, 261.63, 329.63), // Fmaj7 + (196.00, 261.63, 293.66, 392.00) // Gsus4 + }; + + const double secondsPerChord = 4.0; + var dt = 1.0 / SampleRate; + + for (var i = 0; i < frameCount; i++) + { + var t = _sampleTime; + _sampleTime += dt; + + var chordIdx = (int)((t / secondsPerChord) % chords.Length); + var chord = chords[chordIdx]; + + // Harmonic pad synth + var pad1 = Math.Sin(2.0 * Math.PI * chord.c1 * t); + var pad2 = Math.Sin(2.0 * Math.PI * chord.c2 * t); + var pad3 = Math.Sin(2.0 * Math.PI * chord.c3 * t); + var pad4 = Math.Sin(2.0 * Math.PI * chord.c4 * t); + + // Sub bass (centered anchor) + var bass = Math.Sin(2.0 * Math.PI * (chord.c1 * 0.5) * t) * 0.4; + + // Stereo wide acoustic soundstage + var leftChannel = (pad1 * 0.6 + pad3 * 0.5 + bass) * 0.22; + var rightChannel = (pad2 * 0.6 + pad4 * 0.5 + bass) * 0.22; + + // Crisp 16th-note arpeggiator / chime providing high-frequency transients for instant 3D localization + const double arpSpeed = 0.25; // 4 notes per second + var arpNoteIdx = (int)(t / arpSpeed) % 4; + var arpFreq = arpNoteIdx switch + { + 0 => chord.c1 * 2.0, + 1 => chord.c2 * 2.0, + 2 => chord.c3 * 2.0, + _ => chord.c4 * 2.0 + }; + + var arpPhase = (t % arpSpeed) / arpSpeed; + var arpEnv = Math.Exp(-5.0 * arpPhase); + var arpWave = (Math.Sin(2.0 * Math.PI * arpFreq * t) + + 0.35 * Math.Sin(4.0 * Math.PI * arpFreq * t)) * arpEnv * 0.18; + + // Alternate arpeggio notes across left and right virtual speakers + if (arpNoteIdx % 2 == 0) + { + leftChannel += arpWave * 0.85; + rightChannel += arpWave * 0.15; + } + else + { + leftChannel += arpWave * 0.15; + rightChannel += arpWave * 0.85; + } + + // Subtle spatial ping every 2 seconds + var pingPhase = t % 2.0; + if (pingPhase < 0.6) + { + var pingEnv = Math.Exp(-6.0 * pingPhase); + var pingFreq = chord.c3 * 3.0; // ~1200 Hz + var ping = Math.Sin(2.0 * Math.PI * pingFreq * pingPhase) * pingEnv * 0.12; + leftChannel += ping * 0.5; + rightChannel += ping * 0.5; + } + + rawSpan[i * 2] = (float)leftChannel; + rawSpan[i * 2 + 1] = (float)rightChannel; + } + + // Apply real-time 3D Binaural DSP engine rotated by head orientation + _dspEngine.Process(rawSpan, outputBuffer); + + return sampleCount; + } + + public void Dispose() + { + Stop(); + SpatialAudioService.Instance.OrientationUpdated -= OnOrientationUpdated; + } +} diff --git a/GalaxyBudsClient/Utils/Extensions/MathExtensions.cs b/GalaxyBudsClient/Utils/Extensions/MathExtensions.cs index 56aecd9cd..cf6d0ec7a 100644 --- a/GalaxyBudsClient/Utils/Extensions/MathExtensions.cs +++ b/GalaxyBudsClient/Utils/Extensions/MathExtensions.cs @@ -8,7 +8,8 @@ public static class MathExtensions public static (float roll, float pitch, float yaw) ToRollPitchYaw(this Quaternion q) { var roll = (float) Math.Atan2(2.0 * (q.Z * q.Y + q.W * q.X) , 1.0 - 2.0 * (q.X * q.X + q.Y * q.Y)); - var pitch = (float) Math.Asin(2.0 * (q.Y * q.W - q.Z * q.X)); + var sinp = Math.Clamp(2.0 * (q.Y * q.W - q.Z * q.X), -1.0, 1.0); + var pitch = (float) Math.Asin(sinp); var yaw = (float) Math.Atan2(2.0 * (q.Z * q.W + q.X * q.Y) , - 1.0 + 2.0 * (q.W * q.W + q.X * q.X)); return (roll, pitch, yaw); } diff --git a/GalaxyBudsClient/i18n/br.axaml b/GalaxyBudsClient/i18n/br.axaml index 17775f1a8..07dbf2fba 100644 --- a/GalaxyBudsClient/i18n/br.axaml +++ b/GalaxyBudsClient/i18n/br.axaml @@ -737,4 +737,8 @@ Detalhes: Envia pacotes UDP FreeTrack / OpenTrack para localhost:4242 para simuladores de voo/corrida Rastreamento ativo Rastreamento desativado + Largura do Palco Sonoro Virtual + Ajusta a abertura estéreo dos alto-falantes frontais virtuais + Ambiência de Sala + Simula reflexões acústicas de estúdio para projetar o som fora da cabeça \ No newline at end of file diff --git a/GalaxyBudsClient/i18n/en.axaml b/GalaxyBudsClient/i18n/en.axaml index 1f178ebef..a50aacf32 100644 --- a/GalaxyBudsClient/i18n/en.axaml +++ b/GalaxyBudsClient/i18n/en.axaml @@ -750,4 +750,8 @@ The name change may not be detected by devices (including this one) until you un Sends FreeTrack / OpenTrack UDP packets to localhost:4242 for flight/driving sims Tracking active Tracking stopped + Virtual Soundstage Width + Adjusts the stereo angle of the virtual front speakers + Room Ambience + Simulates studio acoustic reflections to externalize sound \ No newline at end of file diff --git a/GalaxyBudsClient/i18n/pt.axaml b/GalaxyBudsClient/i18n/pt.axaml index 36c9c0310..08e7fbbcf 100644 --- a/GalaxyBudsClient/i18n/pt.axaml +++ b/GalaxyBudsClient/i18n/pt.axaml @@ -737,4 +737,8 @@ Detalhes: Envia pacotes UDP FreeTrack / OpenTrack para localhost:4242 para simuladores de voo/corrida Rastreamento ativo Rastreamento desativado + Largura do Palco Sonoro Virtual + Ajusta a abertura estéreo dos alto-falantes frontais virtuais + Ambiência de Sala + Simula reflexões acústicas de estúdio para projetar o som fora da cabeça \ No newline at end of file From 82070dc5ab64ed843ff34fbbb93f34e9bbec5871 Mon Sep 17 00:00:00 2001 From: Juan Pablo Farias Date: Fri, 4 Sep 2026 16:41:53 -0300 Subject: [PATCH 4/6] refactor: improve spatial audio DSP room acoustics, fix UI slider data binding --- .../Interface/Controls/SettingsSliderItem.cs | 12 ++++----- .../Interface/Pages/SpatialAudioPage.axaml | 4 +-- .../Pages/SpatialAudioPageViewModel.cs | 12 ++++++--- .../SpatialAudio/SpatialAudioDspEngine.cs | 27 ++++++++++++++----- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/GalaxyBudsClient/Interface/Controls/SettingsSliderItem.cs b/GalaxyBudsClient/Interface/Controls/SettingsSliderItem.cs index 9f8d0e383..7e4dcb453 100644 --- a/GalaxyBudsClient/Interface/Controls/SettingsSliderItem.cs +++ b/GalaxyBudsClient/Interface/Controls/SettingsSliderItem.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Timers; using Avalonia; using Avalonia.Controls; @@ -37,19 +37,19 @@ public SettingsSliderItem() private readonly Slider _slider; public static readonly RoutedEvent ValueChangedEvent = - RoutedEvent.Register(nameof(ValueChanged), RoutingStrategies.Bubble); + RoutedEvent.Register(nameof(ValueChanged), RoutingStrategies.Bubble); public static readonly StyledProperty ValueProperty = - AvaloniaProperty.Register(nameof(Value), defaultBindingMode: BindingMode.TwoWay); + AvaloniaProperty.Register(nameof(Value), defaultBindingMode: BindingMode.TwoWay); public static readonly StyledProperty MinimumProperty = - AvaloniaProperty.Register(nameof(Minimum), defaultBindingMode: BindingMode.TwoWay); + AvaloniaProperty.Register(nameof(Minimum), defaultBindingMode: BindingMode.TwoWay); public static readonly StyledProperty MaximumProperty = - AvaloniaProperty.Register(nameof(Maximum), defaultBindingMode: BindingMode.TwoWay); + AvaloniaProperty.Register(nameof(Maximum), defaultBindingMode: BindingMode.TwoWay); public static readonly StyledProperty DebounceProperty = - AvaloniaProperty.Register(nameof(Debounce), defaultBindingMode: BindingMode.OneWay); + AvaloniaProperty.Register(nameof(Debounce), defaultBindingMode: BindingMode.OneWay); public static readonly StyledProperty TickPlacementProperty = Slider.TickPlacementProperty.AddOwner(); diff --git a/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml index 3a5237f76..8791e2554 100644 --- a/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml +++ b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml @@ -144,7 +144,7 @@ Symbol="Speaker2" Minimum="15" Maximum="75" - Value="{Binding SpeakerAngle}" /> + Value="{Binding SpeakerAngle, Mode=TwoWay}" /> + Value="{Binding AmbiencePercent, Mode=TwoWay}" /> diff --git a/GalaxyBudsClient/Interface/ViewModels/Pages/SpatialAudioPageViewModel.cs b/GalaxyBudsClient/Interface/ViewModels/Pages/SpatialAudioPageViewModel.cs index c93671d3e..bc255d720 100644 --- a/GalaxyBudsClient/Interface/ViewModels/Pages/SpatialAudioPageViewModel.cs +++ b/GalaxyBudsClient/Interface/ViewModels/Pages/SpatialAudioPageViewModel.cs @@ -7,7 +7,9 @@ using GalaxyBudsClient.Interface.Pages; using GalaxyBudsClient.Model; using GalaxyBudsClient.Platform.SpatialAudio; +using System.Threading.Tasks; using ReactiveUI.SourceGenerators; +using Serilog; namespace GalaxyBudsClient.Interface.ViewModels.Pages; @@ -24,8 +26,8 @@ public partial class SpatialAudioPageViewModel : MainPageViewModelBase, IDisposa [Reactive] private bool _isOpenTrackBroadcasting; [Reactive] private bool _isDemoPlaying; [Reactive] private string _statusText = Strings.SpatialTrackingInactive; - [Reactive] private double _speakerAngle = 30.0; - [Reactive] private double _ambiencePercent = 12.0; + [Reactive] private int _speakerAngle = 30; + [Reactive] private int _ambiencePercent = 12; public SpatialAudioPageViewModel() { @@ -36,8 +38,8 @@ public SpatialAudioPageViewModel() IsTrackingEnabled = SpatialAudioService.Instance.IsActive; IsDemoPlaying = SpatialMediaPlayer.Instance.IsPlaying; - SpeakerAngle = SpatialMediaPlayer.Instance.VirtualSpeakerAngle; - AmbiencePercent = Math.Round(SpatialMediaPlayer.Instance.AmbienceAmount * 100.0); + SpeakerAngle = (int)Math.Round(SpatialMediaPlayer.Instance.VirtualSpeakerAngle); + AmbiencePercent = (int)Math.Round(SpatialMediaPlayer.Instance.AmbienceAmount * 100.0); UpdateStatusText(); } @@ -90,10 +92,12 @@ private void OnSelfPropertyChanged(object? sender, PropertyChangedEventArgs e) case nameof(SpeakerAngle): SpatialMediaPlayer.Instance.VirtualSpeakerAngle = (float)SpeakerAngle; + Log.Debug("SpatialAudioPageViewModel: SpeakerAngle updated to {Angle}°", SpeakerAngle); break; case nameof(AmbiencePercent): SpatialMediaPlayer.Instance.AmbienceAmount = (float)(AmbiencePercent / 100.0); + Log.Debug("SpatialAudioPageViewModel: AmbiencePercent updated to {Ambience}%", AmbiencePercent); break; } } diff --git a/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs index 74cf1282a..3d05095c1 100644 --- a/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs +++ b/GalaxyBudsClient/Platform/SpatialAudio/SpatialAudioDspEngine.cs @@ -135,13 +135,26 @@ public void Process(ReadOnlySpan input, Span output) sampleRL = ApplyHeadShadowFilter(sampleRL, cutoffR_to_L, ref _filterRL) * gainR_to_L; sampleRR = ApplyHeadShadowFilter(sampleRR, cutoffR_to_R, ref _filterRR) * gainR_to_R; - // Subtle early reflections to simulate studio room acoustic cues - var earlyRefL = ReadFractionalDelay(_delayBufferLeftIn, _writeIndex, delayL_to_L + 120.0f) * AmbienceAmount; - var earlyRefR = ReadFractionalDelay(_delayBufferRightIn, _writeIndex, delayR_to_R + 140.0f) * AmbienceAmount; - - // Sum outputs for Left Ear and Right Ear - var outL = sampleLL + sampleRL + (earlyRefR * 0.5f); - var outR = sampleLR + sampleRR + (earlyRefL * 0.5f); + // Multi-tap room early reflection cluster simulating acoustic walls & floor: + // Tap 1: Cross-wall early bounce (~3.6 ms, 160 samples) + // Tap 2: Rear wall reflection (~8.2 ms, 360 samples) + // Tap 3: Floor / ceiling reflection (~15 ms, 660 samples) + var refl1_L = ReadFractionalDelay(_delayBufferRightIn, _writeIndex, 160.0f); + var refl1_R = ReadFractionalDelay(_delayBufferLeftIn, _writeIndex, 160.0f); + + var refl2_L = ReadFractionalDelay(_delayBufferLeftIn, _writeIndex, 360.0f); + var refl2_R = ReadFractionalDelay(_delayBufferRightIn, _writeIndex, 360.0f); + + var refl3_L = ReadFractionalDelay(_delayBufferRightIn, _writeIndex, 660.0f); + var refl3_R = ReadFractionalDelay(_delayBufferLeftIn, _writeIndex, 660.0f); + + var roomL = ((refl1_L * 0.5f) + (refl2_L * 0.35f) + (refl3_L * 0.25f)) * (AmbienceAmount * 2.5f); + var roomR = ((refl1_R * 0.5f) + (refl2_R * 0.35f) + (refl3_R * 0.25f)) * (AmbienceAmount * 2.5f); + + // Sum outputs for Left Ear and Right Ear with natural acoustic wet/dry balance + var directMix = 1.0f - (AmbienceAmount * 0.35f); + var outL = ((sampleLL + sampleRL) * directMix) + roomL; + var outR = ((sampleLR + sampleRR) * directMix) + roomR; // Soft clipper to prevent any digital distortion output[inIdx] = SoftClip(outL); From b14ed8872e42c5ac53c8abe07160b3664fed7350 Mon Sep 17 00:00:00 2001 From: Juan Pablo Farias Date: Fri, 4 Sep 2026 20:37:28 -0300 Subject: [PATCH 5/6] feat: implement system-wide 360 audio support on macOS using BlackHole virtual driver --- GalaxyBudsClient.Tests/SpatialAudioTests.cs | 12 + .../Interface/Pages/SpatialAudioPage.axaml | 106 +++++++ .../Pages/SpatialAudioPageViewModel.cs | 83 ++++++ .../Platform/SpatialAudio/BlackHoleHelper.cs | 177 ++++++++++++ .../SpatialAudio/CoreAudioDeviceHelper.cs | 156 +++++++++++ .../SpatialAudio/MacOsSpatialAudioCapture.cs | 261 ++++++++++++++++++ .../SpatialAudio/MacOsSpatialAudioSink.cs | 41 +++ .../SpatialSystemAudioStreamer.cs | 195 +++++++++++++ GalaxyBudsClient/i18n/br.axaml | 12 + GalaxyBudsClient/i18n/en.axaml | 12 + GalaxyBudsClient/i18n/pt.axaml | 12 + 11 files changed, 1067 insertions(+) create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/BlackHoleHelper.cs create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/CoreAudioDeviceHelper.cs create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/MacOsSpatialAudioCapture.cs create mode 100644 GalaxyBudsClient/Platform/SpatialAudio/SpatialSystemAudioStreamer.cs diff --git a/GalaxyBudsClient.Tests/SpatialAudioTests.cs b/GalaxyBudsClient.Tests/SpatialAudioTests.cs index 13ccab955..0426b2a34 100644 --- a/GalaxyBudsClient.Tests/SpatialAudioTests.cs +++ b/GalaxyBudsClient.Tests/SpatialAudioTests.cs @@ -197,5 +197,17 @@ public void DspEngine_TurnHeadLeft_ShiftsSoundEnergyToRightEar() // Right ear must receive significantly more energy than left ear (head shadow + ILD) energyR.Should().BeGreaterThan(energyL * 1.5f); } + + [Test] + public void CoreAudioDeviceHelper_FindOutputDeviceUid_ReturnsBudsUidWhenConnected() + { + if (!System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX)) + return; + + var uid = CoreAudioDeviceHelper.FindOutputDeviceUid("Buds"); + // On this Mac with Galaxy Buds2 Pro connected, UID must be found + uid.Should().NotBeNullOrEmpty(); + uid.Should().Contain(":output"); + } } diff --git a/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml index 8791e2554..ff81de0ae 100644 --- a/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml +++ b/GalaxyBudsClient/Interface/Pages/SpatialAudioPage.axaml @@ -172,6 +172,112 @@ IsEnabled="{Binding IsTrackingEnabled}" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +