Expose Frame Packet Trailer (FrameMetadata) for video (#286)

* Expose FrameMetadata send/receive for video frames

Surfaces the Rust FFI Frame Packet Trailer feature in the Unity SDK:
attach FrameMetadata (user_timestamp, frame_id) to outgoing frames via
RtcVideoSource.MetadataProvider, and read it off VideoFrame.Metadata on
received frames. PacketTrailerFeatures on TrackPublishOptions is already
proto-passthrough so no wrapper change is needed there.

Adds a PlayMode E2E test that publishes with both packet-trailer features
enabled, attaches known metadata to outgoing frames, and asserts the
subscriber sees matching frame_id and user_timestamp.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Add TrackPublishOptionsExtensions.WithPacketTrailerFeatures

Provides a Google.Protobuf-free entry point for enabling packet trailer
features on TrackPublishOptions. Required because the LiveKit SDK ships
Google.Protobuf.dll as an explicitly-referenced plugin (deliberate, to
avoid conflicts with com.unity.ai.assistant), so a Unity project's
default Assembly-CSharp cannot use the proto's repeated-field collection
initializer or call .Add on PacketTrailerFeatures directly.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Use frame metadata in the Meet sample

Publishes the local camera with PTF_USER_TIMESTAMP and PTF_FRAME_ID
packet trailers and attaches a monotonic frame_id plus UnixTimeUs
user_timestamp to each outgoing frame via the new MetadataProvider hook.

Logs received trailer values (throttled to ~1 second per stream) for
both main and extra-video tiles, so the Unity console / on-screen
ScrollingLog shows the metadata round-tripping from remote publishers.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Consolidate StubVideoSource and MetadataTestVideoSource into TestVideoSource

Single test helper with a pushFrames flag covers both modes: signaling-only
publication propagation (default) and continuous media flow for round-trip
tests.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Remove meet sample changes

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: MaxHeimbrock

Runtime/Scripts/Participant.cs

@@ -1001,4 +1001,21 @@ public LocalDataTrack Track
 
         public PublishDataTrackError Error { get; private set; }
     }
+
+    /// Helpers for setting <see cref="TrackPublishOptions"/> fields whose underlying type
+    /// lives in Google.Protobuf (e.g. RepeatedField&lt;T&gt;). Unity's default Assembly-CSharp
+    /// does not auto-reference Google.Protobuf, so callers without an asmdef cannot use a
+    /// collection initializer or call <c>.Add</c> on the repeated field directly. These
+    /// helpers keep Google.Protobuf types out of the caller's signature.
+    public static class TrackPublishOptionsExtensions
+    {
+        public static TrackPublishOptions WithPacketTrailerFeatures(
+            this TrackPublishOptions options,
+            params PacketTrailerFeature[] features)
+        {
+            foreach (var feature in features)
+                options.PacketTrailerFeatures.Add(feature);
+            return options;
+        }
+    }
 }

Runtime/Scripts/RtcVideoSource.cs

@@ -30,6 +30,12 @@ public enum VideoStreamSource
         /// Called when we receive a new texture (first texture or the resolution changed)
         public event TextureReceiveDelegate TextureReceived;
 
+        public delegate FrameMetadata FrameMetadataDelegate();
+        /// Invoked once per outgoing frame. Return null (default) to send no trailer.
+        /// To actually serialize the trailer onto RTP, also enable the matching
+        /// PacketTrailerFeatures on the TrackPublishOptions used at publish time.
+        public FrameMetadataDelegate MetadataProvider { get; set; }
+
         protected Texture2D _previewTexture;
         protected NativeArray<byte> _captureBuffer;
         protected VideoStreamSource _sourceType;
@@ -175,6 +181,8 @@ protected virtual bool SendFrame()
                 var now = DateTimeOffset.UtcNow;
                 capture.TimestampUs = now.ToUnixTimeMilliseconds() * 1000 + (now.Ticks % TimeSpan.TicksPerMillisecond) / 10;
                 capture.Buffer = buffer;
+                var metadata = MetadataProvider?.Invoke();
+                if (metadata != null) capture.Metadata = metadata;
                 using var response = request.Send();
                 _reading = false;
                 _requestPending = false;

Runtime/Scripts/VideoFrame.cs

@@ -12,12 +12,14 @@ public sealed class VideoFrame
 
         public long Timestamp;
         public VideoRotation Rotation;
+        public FrameMetadata Metadata;
 
-        public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation)
+        public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation, FrameMetadata metadata = null)
         {
             _info = info;
             Timestamp = timeStamp;
             Rotation = rotation;
+            Metadata = metadata;
         }
     }
 

Runtime/Scripts/VideoStream.cs

@@ -242,7 +242,7 @@ private void OnVideoStreamEvent(VideoStreamEvent e)
             // Avoid allocating VideoFrame objects when nobody is observing them.
             if (FrameReceived != null)
             {
-                var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation);
+                var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation, e.FrameReceived.Metadata);
                 FrameReceived.Invoke(frame);
             }
         }

Tests/PlayMode/TrackTests.cs

@@ -149,7 +149,7 @@ public IEnumerator RemoteTrackPublication_SetVideoQuality_DoesNotThrow()
             var publisherRoom = context.Rooms[0];
             var subscriberRoom = context.Rooms[1];
 
-            var videoSource = new StubVideoSource();
+            var videoSource = new TestVideoSource();
             var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom);
 
             // Video track uses a stub source that never pushes frames. TrackSubscribed may not
@@ -283,10 +283,10 @@ public IEnumerator RemoteTrackPublication_PublisherDisablesCamera_UpdatesFlagAnd
             var publisherRoom = context.Rooms[0];
             var subscriberRoom = context.Rooms[1];
 
-            var videoSource = new StubVideoSource();
+            var videoSource = new TestVideoSource();
             var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom);
 
-            // StubVideoSource never pushes frames, so TrackSubscribed may not fire on the
+            // TestVideoSource (pushFrames=false) never pushes frames, so TrackSubscribed may not fire on the
             // subscriber. The RemoteTrackPublication still propagates via TrackPublished.
             var publishedExp = new Expectation(timeoutSeconds: 10f);
             subscriberRoom.TrackPublished += (_, _) => publishedExp.Fulfill();

Tests/PlayMode/Utils/CoroutineRunner.cs

@@ -0,0 +1,12 @@
+using UnityEngine;
+
+namespace LiveKit.PlayModeTests.Utils
+{
+    /// <summary>
+    /// Empty MonoBehaviour used by tests to host long-running coroutines
+    /// (e.g. <see cref="RtcVideoSource.Update"/> and <see cref="VideoStream.Update"/>)
+    /// that the test itself cannot host because <c>[UnityTest]</c> bodies must yield
+    /// back to the test runner.
+    /// </summary>
+    public class CoroutineRunner : MonoBehaviour { }
+}

…s/PlayMode/Utils/StubVideoSource.cs.meta …s/PlayMode/Utils/CoroutineRunner.cs.metaTests/PlayMode/Utils/StubVideoSource.cs.meta renamed to Tests/PlayMode/Utils/CoroutineRunner.cs.meta Tests/PlayMode/Utils/StubVideoSource.cs.meta renamed to Tests/PlayMode/Utils/CoroutineRunner.cs.meta

@@ -0,0 +1,53 @@
+using LiveKit.Proto;
+using Unity.Collections;
+
+namespace LiveKit.PlayModeTests.Utils
+{
+    /// <summary>
+    /// Test-only <see cref="RtcVideoSource"/> registered with FFI at a fixed
+    /// resolution. Two modes via <paramref name="pushFrames"/>:
+    /// <list type="bullet">
+    /// <item><c>false</c> (default): never pushes frames. Use when the test only
+    /// needs the publication to propagate via signaling (e.g.
+    /// <see cref="RemoteTrackPublication"/> APIs that operate on metadata).</item>
+    /// <item><c>true</c>: pushes a continuous stream of zero-filled RGBA frames.
+    /// Use when the test needs actual media flow (e.g. validating per-frame
+    /// metadata round-trips through the FFI / RTP path).</item>
+    /// </list>
+    /// </summary>
+    public sealed class TestVideoSource : RtcVideoSource
+    {
+        private readonly int _width;
+        private readonly int _height;
+        private readonly bool _pushFrames;
+
+        public override int GetWidth() => _width;
+        public override int GetHeight() => _height;
+
+        protected override VideoRotation GetVideoRotation() => VideoRotation._0;
+
+        protected override bool ReadBuffer()
+        {
+            if (!_pushFrames) return false;
+
+            if (!_captureBuffer.IsCreated)
+            {
+                _captureBuffer = new NativeArray<byte>(
+                    _width * _height * 4,
+                    Allocator.Persistent,
+                    NativeArrayOptions.ClearMemory);
+            }
+            _requestPending = true;
+            return false;
+        }
+
+        public TestVideoSource(bool pushFrames = false, int width = 16, int height = 16)
+            : base(VideoStreamSource.Texture, VideoBufferType.Rgba)
+        {
+            _width = width;
+            _height = height;
+            _pushFrames = pushFrames;
+            Init();
+        }
+    }
+}