Skip to content

Commit 9f5a45a

Browse files
YimingZhanshenYimingZhanshen
authored
Merge pull request #4 from YimingZhanshen/copilot/improve-audio-synchronization
Fix screen mirroring audio, video freeze on reconnect, volume control, and improve lossless codec support
2 parents ddd6b58 + a5af23f commit 9f5a45a

13 files changed

Lines changed: 838 additions & 179 deletions

‎AirPlay/AirPlayReceiver.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public Task StartMdnsAsync()
8282
// Internally 'ServiceProfile' create the SRV record
8383
var airTunes = new ServiceProfile($"{deviceIdInstance}@{_instance}", AirTunesType, _airTunesPort);
8484
airTunes.AddProperty("ch", "2");
85-
airTunes.AddProperty("cn", "2,3");
85+
airTunes.AddProperty("cn", "0,1,2,3");
8686
airTunes.AddProperty("et", "0,3,5");
8787
airTunes.AddProperty("md", "0,1,2");
8888
airTunes.AddProperty("sr", "44100");

‎AirPlay/AirPlayService.cs‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ public class AirPlayService : IHostedService, IDisposable
1818
private readonly DumpConfig _dConfig;
1919

2020
private AudioOutputService _audioOutput;
21+
private VideoOutputService _videoOutput;
2122
private List<byte> _audiobuf;
2223
private readonly object _audioOutputLock = new object();
24+
private readonly object _videoOutputLock = new object();
2325

2426
public AirPlayService(IAirPlayReceiver airPlayReceiver, IOptions<DumpConfig> dConfig)
2527
{
@@ -94,12 +96,28 @@ public async Task StartAsync(CancellationToken cancellationToken)
9496
}
9597
}
9698

99+
// Initialize video output (named pipe for external player)
100+
try
101+
{
102+
_videoOutput = new VideoOutputService();
103+
_videoOutput.Initialize();
104+
Console.WriteLine("Video output initialized successfully");
105+
}
106+
catch (Exception ex)
107+
{
108+
Console.WriteLine($"Failed to initialize video output: {ex.Message}");
109+
Console.WriteLine("Continuing without video output...");
110+
}
111+
97112
await _airPlayReceiver.StartListeners(cancellationToken);
98113
await _airPlayReceiver.StartMdnsAsync().ConfigureAwait(false);
99114

100115
_airPlayReceiver.OnSetVolumeReceived += (s, e) =>
101116
{
102-
// SET VOLUME
117+
lock (_audioOutputLock)
118+
{
119+
_audioOutput?.SetVolume(e);
120+
}
103121
};
104122

105123
_airPlayReceiver.OnAudioFlushReceived += (s, e) =>
@@ -111,10 +129,15 @@ public async Task StartAsync(CancellationToken cancellationToken)
111129
}
112130
};
113131

114-
// DUMP H264 VIDEO
132+
// H264 VIDEO OUTPUT
115133
_airPlayReceiver.OnH264DataReceived += (s, e) =>
116134
{
117-
// DO SOMETHING WITH VIDEO DATA..
135+
// Send H264 data to video output pipe
136+
lock (_videoOutputLock)
137+
{
138+
_videoOutput?.WriteFrame(e);
139+
}
140+
118141
#if DUMP
119142
using (FileStream writer = new FileStream($"{bPath}dump.h264", FileMode.Append))
120143
{
@@ -143,6 +166,9 @@ public Task StopAsync(CancellationToken cancellationToken)
143166
_audioOutput?.Dispose();
144167
_audioOutput = null;
145168

169+
_videoOutput?.Dispose();
170+
_videoOutput = null;
171+
146172
#if DUMP
147173
// DUMP WAV AUDIO
148174
var bPath = _dConfig.Path;
@@ -165,6 +191,9 @@ public void Dispose()
165191
_audioOutput?.Dispose();
166192
_audioOutput = null;
167193

194+
_videoOutput?.Dispose();
195+
_videoOutput = null;
196+
168197
if (_airPlayReceiver is IDisposable disposable)
169198
{
170199
disposable.Dispose();

‎AirPlay/Decoders/Implementations/FFmpegAacEldDecoder.cs‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
/*
2+
* AAC-ELD Decoder using FFmpeg as a subprocess.
3+
*
4+
* The fdk-aac NuGet package returns error 0x5 (AAC_DEC_UNSUPPORTED_ER_FORMAT)
5+
* for AAC-ELD because the pre-built binary doesn't include ER format support.
6+
* SharpJaad.AAC also doesn't support AAC-ELD.
7+
*
8+
* This decoder pipes raw AAC-ELD frames (wrapped in ADTS) through FFmpeg for
9+
* decoding, which supports AAC-ELD natively. This approach works with any
10+
* FFmpeg version the user has installed.
11+
*
12+
* Reference: UxPlay uses GStreamer's avdec_aac (which wraps FFmpeg) for the same purpose.
13+
*/
14+
15+
using System;
16+
using System.Diagnostics;
17+
using System.IO;
18+
using System.Threading;
19+
using AirPlay.Models.Enums;
20+
21+
namespace AirPlay.Decoders.Implementations
22+
{
23+
public class FFmpegAacEldDecoder : IDecoder, IDisposable
24+
{
25+
private const int MAX_READ_ATTEMPTS = 50;
26+
private const int PROCESS_EXIT_TIMEOUT_MS = 1000;
27+
28+
private Process _ffmpegProcess;
29+
private BinaryWriter _ffmpegInput;
30+
private Stream _ffmpegOutput;
31+
private int _pcmOutputSize;
32+
private int _channels;
33+
private int _sampleRate;
34+
private int _frameLength;
35+
private bool _disposed;
36+
private bool _initialized;
37+
private readonly byte[] _adtsHeader = new byte[7];
38+
private int _adtsProfile;
39+
private int _adtsFreqIdx;
40+
private int _adtsChanCfg;
41+
42+
public AudioFormat Type => AudioFormat.AAC_ELD;
43+
44+
public FFmpegAacEldDecoder()
45+
{
46+
}
47+
48+
public int Config(int sampleRate, int channels, int bitDepth, int frameLength)
49+
{
50+
_channels = channels;
51+
_sampleRate = sampleRate;
52+
_frameLength = frameLength;
53+
_pcmOutputSize = frameLength * channels * (bitDepth / 8);
54+
55+
// ADTS profile=2 (AAC-LC) since ADTS doesn't support AAC-ELD profile encoding.
56+
// FFmpeg's parser will still decode the content correctly based on the actual bitstream.
57+
_adtsProfile = 2;
58+
_adtsFreqIdx = GetSampleRateIndex(sampleRate);
59+
_adtsChanCfg = channels;
60+
61+
try
62+
{
63+
// Start FFmpeg process:
64+
// Input: AAC frames wrapped in ADTS headers via stdin pipe
65+
// Output: raw PCM S16LE via stdout pipe
66+
var psi = new ProcessStartInfo
67+
{
68+
FileName = "ffmpeg",
69+
Arguments = $"-hide_banner -loglevel error -f aac -i pipe:0 -f s16le -acodec pcm_s16le -ar {sampleRate} -ac {channels} pipe:1",
70+
UseShellExecute = false,
71+
RedirectStandardInput = true,
72+
RedirectStandardOutput = true,
73+
RedirectStandardError = true,
74+
CreateNoWindow = true,
75+
};
76+
77+
_ffmpegProcess = new Process { StartInfo = psi };
78+
_ffmpegProcess.Start();
79+
80+
_ffmpegInput = new BinaryWriter(_ffmpegProcess.StandardInput.BaseStream);
81+
_ffmpegOutput = _ffmpegProcess.StandardOutput.BaseStream;
82+
83+
// Drain stderr in background to prevent pipe deadlock
84+
var stderrThread = new Thread(() =>
85+
{
86+
try { _ffmpegProcess.StandardError.ReadToEnd(); }
87+
catch { /* ignore */ }
88+
});
89+
stderrThread.IsBackground = true;
90+
stderrThread.Start();
91+
92+
_initialized = true;
93+
Console.WriteLine($"FFmpeg AAC-ELD decoder started: {sampleRate}Hz, {channels}ch, {bitDepth}bit, frameLength={frameLength}");
94+
return 0;
95+
}
96+
catch (Exception ex)
97+
{
98+
Console.WriteLine($"FFmpeg AAC-ELD decoder failed to start: {ex.Message}");
99+
Console.WriteLine("Please ensure 'ffmpeg' is in your system PATH.");
100+
return -1;
101+
}
102+
}
103+
104+
public int GetOutputStreamLength()
105+
{
106+
return _pcmOutputSize;
107+
}
108+
109+
public int DecodeFrame(byte[] input, ref byte[] output, int length)
110+
{
111+
if (!_initialized || _ffmpegProcess == null || _ffmpegProcess.HasExited)
112+
return -1;
113+
114+
try
115+
{
116+
// Wrap the raw AAC frame in an ADTS header for FFmpeg to parse
117+
int frameLen = input.Length + 7; // ADTS header is 7 bytes
118+
BuildAdtsHeader(_adtsHeader, frameLen, _adtsProfile, _adtsFreqIdx, _adtsChanCfg);
119+
120+
// Write ADTS header + raw AAC data to FFmpeg stdin
121+
_ffmpegInput.Write(_adtsHeader, 0, 7);
122+
_ffmpegInput.Write(input, 0, input.Length);
123+
_ffmpegInput.Flush();
124+
125+
// Read decoded PCM from FFmpeg stdout
126+
int bytesToRead = _pcmOutputSize;
127+
int totalRead = 0;
128+
int maxAttempts = MAX_READ_ATTEMPTS;
129+
130+
while (totalRead < bytesToRead && maxAttempts > 0)
131+
{
132+
int read = _ffmpegOutput.Read(output, totalRead, bytesToRead - totalRead);
133+
if (read <= 0)
134+
{
135+
Thread.Sleep(1);
136+
maxAttempts--;
137+
continue;
138+
}
139+
totalRead += read;
140+
}
141+
142+
if (totalRead < bytesToRead)
143+
{
144+
// Partial read - zero fill the rest
145+
Array.Clear(output, totalRead, bytesToRead - totalRead);
146+
}
147+
148+
return 0;
149+
}
150+
catch (Exception)
151+
{
152+
return -1;
153+
}
154+
}
155+
156+
/// <summary>
157+
/// Build an ADTS header for wrapping raw AAC frames.
158+
/// </summary>
159+
private static void BuildAdtsHeader(byte[] header, int packetLen, int profile, int freqIdx, int chanCfg)
160+
{
161+
header[0] = 0xFF;
162+
header[1] = 0xF1; // MPEG-4, Layer 0, no CRC
163+
header[2] = (byte)(((profile - 1) << 6) | (freqIdx << 2) | (chanCfg >> 2));
164+
header[3] = (byte)(((chanCfg & 3) << 6) | (packetLen >> 11));
165+
header[4] = (byte)((packetLen >> 3) & 0xFF);
166+
header[5] = (byte)(((packetLen & 7) << 5) | 0x1F);
167+
header[6] = 0xFC;
168+
}
169+
170+
private static int GetSampleRateIndex(int sampleRate)
171+
{
172+
return sampleRate switch
173+
{
174+
96000 => 0, 88200 => 1, 64000 => 2, 48000 => 3,
175+
44100 => 4, 32000 => 5, 24000 => 6, 22050 => 7,
176+
16000 => 8, 12000 => 9, 11025 => 10, 8000 => 11,
177+
7350 => 12, _ => 4,
178+
};
179+
}
180+
181+
public void Dispose()
182+
{
183+
if (!_disposed)
184+
{
185+
_disposed = true;
186+
try
187+
{
188+
// Close stdin first for graceful FFmpeg shutdown
189+
_ffmpegInput?.Close();
190+
191+
if (_ffmpegProcess != null && !_ffmpegProcess.HasExited)
192+
{
193+
// Wait for graceful exit, then force kill if needed
194+
if (!_ffmpegProcess.WaitForExit(PROCESS_EXIT_TIMEOUT_MS))
195+
{
196+
_ffmpegProcess.Kill();
197+
_ffmpegProcess.WaitForExit(PROCESS_EXIT_TIMEOUT_MS);
198+
}
199+
}
200+
_ffmpegProcess?.Dispose();
201+
}
202+
catch { /* ignore cleanup errors */ }
203+
}
204+
}
205+
}
206+
}

‎AirPlay/Decoders/Implementations/PCMDecoder.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,26 @@ namespace AirPlay
55
{
66
public class PCMDecoder : IDecoder
77
{
8+
private int _pcmSize;
9+
810
public AudioFormat Type => AudioFormat.PCM;
911

1012
public int Config(int sampleRate, int channels, int bitDepth, int frameLength)
1113
{
14+
_pcmSize = frameLength * channels * (bitDepth / 8);
1215
return 0;
1316
}
1417

1518
public int DecodeFrame(byte[] input, ref byte[] output, int length)
1619
{
17-
Array.Copy(input, 0, output, 0, input.Length);
20+
int copyLen = Math.Min(input.Length, output.Length);
21+
Array.Copy(input, 0, output, 0, copyLen);
1822
return 0;
1923
}
2024

2125
public int GetOutputStreamLength()
2226
{
23-
return -1;
27+
return _pcmSize > 0 ? _pcmSize : 1024 * 4; // default: 1024 samples * stereo * 16-bit
2428
}
2529
}
2630
}

‎AirPlay/Listeners/AirTunesListener.cs‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,12 +333,20 @@ public override async Task OnDataReceivedAsync(Request request, Response respons
333333
{
334334
if (stream.ContainsKey("audioFormat"))
335335
{
336-
var audioFormat = (int)stream["audioFormat"];
336+
var audioFormat = Convert.ToInt32(stream["audioFormat"]);
337337
session.AudioFormat = (AudioFormat)audioFormat;
338338

339339
var description = GetAudioFormatDescription(audioFormat);
340340
Console.WriteLine($"Audio type: {description}");
341341
}
342+
if (stream.ContainsKey("ct"))
343+
{
344+
session.AudioCompressionType = Convert.ToInt32(stream["ct"]);
345+
}
346+
if (stream.ContainsKey("spf"))
347+
{
348+
session.AudioSamplesPerFrame = Convert.ToInt32(stream["spf"]);
349+
}
342350
if (stream.ContainsKey("controlPort"))
343351
{
344352
// Use this port to request resend lost packet? (remote port)
@@ -440,8 +448,17 @@ public override async Task OnDataReceivedAsync(Request request, Response respons
440448

441449
session.StreamingListener = streaming;
442450
}
443-
if (session.FairPlayReady && session.AudioSessionReady && session.AudioControlListener == null)
451+
if (session.FairPlayReady && session.AudioSessionReady)
444452
{
453+
// Stop existing audio listener before creating a new one
454+
// (ports 7002/7003 must be released first)
455+
if (session.AudioControlListener != null)
456+
{
457+
try { await session.AudioControlListener.StopAsync(); }
458+
catch (Exception ex) { Console.WriteLine($"Error stopping old audio listener: {ex.Message}"); }
459+
session.AudioControlListener = null;
460+
}
461+
445462
// Start 'AudioListener' (handle PCM/AAC/ALAC data received from iOS/macOS
446463
var control = new AudioListener(_receiver, session.SessionId, 7002, 7003, _dumpConfig);
447464
await control.StartAsync(cancellationToken).ConfigureAwait(false);
@@ -565,7 +582,12 @@ public override async Task OnDataReceivedAsync(Request request, Response respons
565582
if (session.MirroringListener != null)
566583
{
567584
await session.MirroringListener.StopAsync();
585+
session.MirroringListener = null;
568586
}
587+
// Reset video state for clean reconnect
588+
session.SpsPps = null;
589+
session.StreamConnectionId = null;
590+
session.MirroringSession = null;
569591
}
570592
// If audio session
571593
if (type == 96)
@@ -574,7 +596,9 @@ public override async Task OnDataReceivedAsync(Request request, Response respons
574596
if (session.AudioControlListener != null)
575597
{
576598
await session.AudioControlListener.StopAsync();
599+
session.AudioControlListener = null;
577600
}
601+
session.AudioFormat = AudioFormat.Unknown;
578602
}
579603
}
580604
}
@@ -593,6 +617,9 @@ private string GetAudioFormatDescription(int format)
593617

594618
switch (format)
595619
{
620+
case 0x0:
621+
formatDescription = "96 L16/44100/2 (PCM)";
622+
break;
596623
case 0x40000:
597624
formatDescription = "96 AppleLossless, 96 352 0 16 40 10 14 2 255 0 0 44100";
598625
break;

0 commit comments

Comments
 (0)