Add zero-copy video frames and surface dropped frames

include/livekit/video_frame.h

@@ -21,6 +21,7 @@
 #include <optional>
 #include <vector>
 
+#include "livekit/ffi_handle.h"
 #include "livekit/visibility.h"
 
 namespace livekit {
@@ -41,10 +42,21 @@ class OwnedVideoBuffer;
 
 /// Public SDK representation of a video frame.
 ///
-/// - Owns its pixel buffer (std::vector<uint8_t>).
+/// - Owns its pixel buffer (std::vector<uint8_t>), unless borrowed.
 /// - Developers can allocate and fill frames in C++ and pass them to the SDK.
 /// - The SDK can expose the backing memory to Rust via data_ptr + layout for
 ///   the duration of a blocking FFI call (similar to AudioFrame).
+///
+/// ## Borrowed frames
+///
+/// A frame delivered by a VideoStream opened with `Options::zero_copy` reads the SDK's own buffer
+/// instead of a copy of it, saving a full frame-sized memcpy per frame. Such a frame holds the FFI
+/// handle that keeps the buffer alive, so the buffer is returned when the frame is destroyed or
+/// release() is called -- plain RAII, and the reason VideoFrame is move-only.
+///
+/// Every accessor works the same either way. The difference is lifetime: pointers from data() and
+/// planeInfos() on a borrowed frame are only valid while that frame is, so do not stash them past
+/// its scope.
 class LIVEKIT_API VideoFrame {
 public:
   VideoFrame();
@@ -53,8 +65,10 @@ class LIVEKIT_API VideoFrame {
 
   VideoFrame(const VideoFrame&) = delete;
   VideoFrame& operator=(const VideoFrame&) = delete;
-  VideoFrame(VideoFrame&&) noexcept = default;
-  VideoFrame& operator=(VideoFrame&&) noexcept = default;
+  // Hand-written rather than defaulted: a defaulted move copies the borrowed pointers and leaves
+  // the source naming memory it no longer owns.
+  VideoFrame(VideoFrame&&) noexcept;
+  VideoFrame& operator=(VideoFrame&&) noexcept;
 
   /// Allocate a new frame with the correct buffer size for the given format.
   /// Data is zero-initialized.
@@ -65,14 +79,28 @@ class LIVEKIT_API VideoFrame {
   int height() const noexcept { return height_; }
   VideoBufferType type() const noexcept { return type_; }
 
-  std::uint8_t* data() noexcept { return data_.data(); }
-  const std::uint8_t* data() const noexcept { return data_.data(); }
-  std::size_t dataSize() const noexcept { return data_.size(); }
+  std::uint8_t* data() noexcept { return borrowed_data_ != nullptr ? borrowed_data_ : data_.data(); }
+  const std::uint8_t* data() const noexcept { return borrowed_data_ != nullptr ? borrowed_data_ : data_.data(); }
+  std::size_t dataSize() const noexcept { return borrowed_data_ != nullptr ? borrowed_size_ : data_.size(); }
+
+  /// Whether this frame reads the SDK's buffer rather than owning a copy.
+  /// See the class comment for what that means for pointer lifetimes.
+  bool isBorrowed() const noexcept { return borrowed_data_ != nullptr; }
+
+  /// Returns a borrowed frame's buffer to the SDK early, instead of waiting for the destructor.
+  /// No-op on an owning frame, and safe to call twice.
+  ///
+  /// After this, data() and planeInfos() describe an empty frame. Anything still pointing into the
+  /// old buffer is dangling.
+  void release() noexcept;
 
   /// Compute plane layout for this frame (Y/U/V, UV, etc.), in terms of
   /// pointers & sizes relative to this frame's backing buffer.
   ///
   /// For packed formats (ARGB, RGB24) this will be either 1 plane or empty.
+  ///
+  /// A borrowed frame reports the layout the SDK gave it, rather than one derived from
+  /// width/height, so a padded stride is described accurately.
   std::vector<VideoPlaneInfo> planeInfos() const;
 
   /// Convert this frame into another pixel format.
@@ -104,13 +132,28 @@ class LIVEKIT_API VideoFrame {
   friend class VideoStream;
   // Only internal classes (e.g., VideoStream)
   // should construct frames directly from FFI buffers.
+
+  /// Copies the pixels out of the SDK buffer and releases the handle.
   static VideoFrame fromOwnedInfo(const proto::OwnedVideoBuffer& owned);
 
+  /// Wraps the SDK buffer in place and takes ownership of the handle, so the buffer lives exactly
+  /// as long as the returned frame. No copy.
+  static VideoFrame borrowOwnedInfo(const proto::OwnedVideoBuffer& owned);
+
 private:
   int width_;
   int height_;
   VideoBufferType type_;
+
+  /// Pixels for an owning frame; empty for a borrowed one.
   std::vector<std::uint8_t> data_;
+
+  /// Non-null only for a borrowed frame, in which case it points into the buffer
+  /// `borrowed_handle_` keeps alive. Doubles as the "is borrowed" flag.
+  std::uint8_t* borrowed_data_{nullptr};
+  std::size_t borrowed_size_{0};
+  std::vector<VideoPlaneInfo> borrowed_planes_;
+  FfiHandle borrowed_handle_;
 };
 
 } // namespace livekit

include/livekit/video_stream.h

@@ -43,6 +43,17 @@ struct VideoFrameEvent {
   std::int64_t timestamp_us;
   VideoRotation rotation;
   std::optional<VideoFrameMetadata> metadata;
+
+  /// Frames this stream decoded but never delivered, counted since the stream was created.
+  ///
+  /// A frame is dropped when the SDK's internal queue overflows because this application did not
+  /// read frames fast enough. No WebRTC stat reports the loss: `framesDecoded` counts these frames
+  /// and `framesDropped` does not, so this counter is the only way to observe it.
+  ///
+  /// Cumulative and monotonic: compare against the previous event's value to see how many frames
+  /// went missing in between. Counts losses on both sides of the FFI -- frames the SDK's own
+  /// queue overflowed, and frames this stream's `capacity` evicted before `read()` reached them.
+  std::uint64_t dropped_frames{0};
 };
 
 namespace proto {
@@ -77,6 +88,17 @@ class LIVEKIT_API VideoStream {
     /// Preferred pixel format for frames delivered by read(). The FFI layer
     /// converts into this format if supported (e.g., RGBA, BGRA, I420, ...).
     VideoBufferType format{VideoBufferType::RGBA};
+
+    /// Deliver frames that read the SDK's buffer instead of a copy of it.
+    ///
+    /// Saves a full frame-sized memcpy per frame -- 4.5 MB at 1628x1920 I420, so 134 MB/s at
+    /// 30 fps. In exchange the frame's lifetime starts to matter: its buffer is returned to the
+    /// SDK when the VideoFrame is destroyed, so pointers taken from data() or planeInfos() must
+    /// not outlive it. See VideoFrame's class comment.
+    ///
+    /// Prefer a non-zero `capacity` alongside this. Queued frames hold SDK buffers, so an
+    /// unbounded queue plus a slow reader grows native memory.
+    bool zero_copy{false};
   };
 
   /// Factory: create a VideoStream bound to a specific Track
@@ -126,6 +148,9 @@ class LIVEKIT_API VideoStream {
   std::condition_variable cv_;
   std::deque<VideoFrameEvent> queue_;
   std::size_t capacity_{0};
+  bool zero_copy_{false};
+  // Frames this stream's own queue evicted; added to the SDK's count on delivery.
+  std::uint64_t evicted_frames_{0};
   bool eof_{false};
   bool closed_{false};
 

src/tests/integration/test_video_stream_zero_copy.cpp

@@ -0,0 +1,239 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// End-to-end cover for VideoStream::Options::zero_copy.
+//
+// The unit tests build the OwnedVideoBuffer by hand; these publish a real
+// track, let it travel through the SFU and the decoder, and read it back with
+// a borrowed frame. That is the only way to see the pointers the FFI actually
+// hands over, and the only way to know the buffer stays valid across the
+// pooled-buffer recycling the Rust side now does.
+
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "tests/common/test_common.h"
+#include "tests/common/video_utils.h"
+
+namespace livekit::test {
+namespace {
+
+using namespace std::chrono_literals;
+
+constexpr int kWidth = 320;
+constexpr int kHeight = 240;
+
+/// Publishes frames until stopped, so the subscriber has something to read.
+class FramePublisher {
+public:
+  FramePublisher(std::shared_ptr<VideoSource> source, int width, int height)
+      : source_(std::move(source)), width_(width), height_(height) {}
+
+  void start() {
+    thread_ = std::thread([this] {
+      std::uint64_t n = 0;
+      while (!stop_) {
+        VideoFrame frame = VideoFrame::create(width_, height_, VideoBufferType::ARGB);
+        fillWebcamLikeFrame(frame, n++);
+        source_->captureFrame(frame);
+        std::this_thread::sleep_for(33ms); // ~30 fps
+      }
+    });
+  }
+
+  ~FramePublisher() {
+    stop_ = true;
+    if (thread_.joinable()) {
+      thread_.join();
+    }
+  }
+
+private:
+  std::shared_ptr<VideoSource> source_;
+  int width_;
+  int height_;
+  std::atomic<bool> stop_{false};
+  std::thread thread_;
+};
+
+/// Finds the remote video publication named `track_name`, waiting for it.
+std::shared_ptr<Track> awaitRemoteVideoTrack(Room& room, const std::string& identity, const std::string& track_name,
+                                             std::chrono::seconds timeout) {
+  const auto deadline = std::chrono::steady_clock::now() + timeout;
+  while (std::chrono::steady_clock::now() < deadline) {
+    auto participant = room.remoteParticipant(identity).lock();
+    if (participant != nullptr) {
+      for (const auto& [sid, publication] : participant->trackPublications()) {
+        (void)sid;
+        if (publication == nullptr || publication->kind() != TrackKind::KIND_VIDEO ||
+            publication->name() != track_name) {
+          continue;
+        }
+        if (auto track = publication->track(); track != nullptr) {
+          return track;
+        }
+      }
+    }
+    std::this_thread::sleep_for(100ms);
+  }
+  return nullptr;
+}
+
+} // namespace
+
+class VideoStreamZeroCopyServerTest : public LiveKitTestBase {
+protected:
+  /// Connects both rooms, publishes a video track, and returns the remote
+  /// track the subscriber sees.
+  std::shared_ptr<Track> publishAndSubscribe() {
+    const std::string track_name = "zero-copy-track";
+
+    EXPECT_TRUE(receiver_room_.connect(config_.url, config_.token_b, RoomOptions{}));
+    EXPECT_TRUE(sender_room_.connect(config_.url, config_.token_a, RoomOptions{}));
+
+    const std::string sender_identity = lockLocalParticipant(sender_room_)->identity();
+    EXPECT_FALSE(sender_identity.empty());
+    EXPECT_TRUE(waitForParticipant(&receiver_room_, sender_identity, 10s));
+
+    source_ = std::make_shared<VideoSource>(kWidth, kHeight);
+    auto track = LocalVideoTrack::createLocalVideoTrack(track_name, source_);
+
+    TrackPublishOptions publish_options;
+    publish_options.source = TrackSource::SOURCE_CAMERA;
+    publish_options.simulcast = false;
+    lockLocalParticipant(sender_room_)->publishTrack(track, publish_options);
+
+    publisher_ = std::make_unique<FramePublisher>(source_, kWidth, kHeight);
+    publisher_->start();
+
+    return awaitRemoteVideoTrack(receiver_room_, sender_identity, track_name, 20s);
+  }
+
+  Room sender_room_;
+  Room receiver_room_;
+  std::shared_ptr<VideoSource> source_;
+  std::unique_ptr<FramePublisher> publisher_;
+};
+
+TEST_F(VideoStreamZeroCopyServerTest, BorrowedFramesCarryDecodedPixels) {
+  failIfNotConfigured();
+
+  auto remote_track = publishAndSubscribe();
+  ASSERT_NE(remote_track, nullptr) << "the subscriber never saw the published video track";
+
+  VideoStream::Options options;
+  options.format = VideoBufferType::I420;
+  options.zero_copy = true;
+  options.capacity = 8;
+  auto stream = VideoStream::fromTrack(remote_track, options);
+  ASSERT_NE(stream, nullptr);
+
+  // Read a run of frames. Each one borrows the SDK's buffer, and the previous
+  // one's buffer has already gone back to the Rust pool to be handed out
+  // again -- so reading valid pixels here is what shows the recycling is safe.
+  constexpr int kFramesToRead = 20;
+  int read = 0;
+  VideoFrameEvent event;
+  while (read < kFramesToRead && stream->read(event)) {
+    ASSERT_TRUE(event.frame.isBorrowed()) << "frame " << read << " was copied, not borrowed";
+    ASSERT_NE(event.frame.data(), nullptr);
+
+    const std::size_t expected = static_cast<std::size_t>(kWidth) * kHeight +
+                                 2 * (static_cast<std::size_t>((kWidth + 1) / 2) * ((kHeight + 1) / 2));
+    EXPECT_EQ(event.frame.dataSize(), expected);
+    EXPECT_EQ(event.frame.width(), kWidth);
+    EXPECT_EQ(event.frame.height(), kHeight);
+
+    const auto planes = event.frame.planeInfos();
+    ASSERT_EQ(planes.size(), 3u) << "I420 should report three planes";
+    for (const auto& plane : planes) {
+      EXPECT_NE(plane.data_ptr, 0u);
+      EXPECT_GT(plane.stride, 0u);
+    }
+
+    // The publisher paints a green-ish frame, so the luma plane must not be
+    // uniformly zero. This is the check that the pointer really addresses
+    // decoded pixels rather than a buffer that was released underneath us.
+    const std::uint8_t* luma = event.frame.data();
+    std::uint64_t luma_sum = 0;
+    for (std::size_t i = 0; i < static_cast<std::size_t>(kWidth) * kHeight; ++i) {
+      luma_sum += luma[i];
+    }
+    EXPECT_GT(luma_sum, 0u) << "frame " << read << " decoded to an all-black luma plane";
+
+    ++read;
+  }
+
+  EXPECT_EQ(read, kFramesToRead) << "the stream ended after " << read << " frames";
+  stream->close();
+}
+
+TEST_F(VideoStreamZeroCopyServerTest, DroppedFrameCountIsReportedAndStartsAtZero) {
+  failIfNotConfigured();
+
+  auto remote_track = publishAndSubscribe();
+  ASSERT_NE(remote_track, nullptr);
+
+  VideoStream::Options options;
+  options.format = VideoBufferType::I420;
+  options.capacity = 8;
+  auto stream = VideoStream::fromTrack(remote_track, options);
+  ASSERT_NE(stream, nullptr);
+
+  // A reader that keeps up should see no drops, and the counter must never
+  // run backwards: it is cumulative over the stream's life.
+  std::uint64_t previous = 0;
+  VideoFrameEvent event;
+  for (int i = 0; i < 20 && stream->read(event); ++i) {
+    EXPECT_GE(event.dropped_frames, previous) << "the counter went backwards at frame " << i;
+    previous = event.dropped_frames;
+  }
+
+  EXPECT_EQ(previous, 0u) << "a reader keeping up with the stream should not lose frames";
+  stream->close();
+}
+
+TEST_F(VideoStreamZeroCopyServerTest, OwningFramesOutliveTheStream) {
+  failIfNotConfigured();
+
+  auto remote_track = publishAndSubscribe();
+  ASSERT_NE(remote_track, nullptr);
+
+  VideoStream::Options options;
+  options.format = VideoBufferType::I420;
+  options.zero_copy = false; // the default: each frame owns a copy
+  options.capacity = 8;
+  auto stream = VideoStream::fromTrack(remote_track, options);
+  ASSERT_NE(stream, nullptr);
+
+  VideoFrameEvent event;
+  ASSERT_TRUE(stream->read(event));
+  ASSERT_FALSE(event.frame.isBorrowed());
+  const std::vector<std::uint8_t> snapshot(event.frame.data(), event.frame.data() + event.frame.dataSize());
+
+  stream->close();
+  stream.reset();
+
+  // An owning frame is independent of the stream, so its pixels survive.
+  ASSERT_EQ(event.frame.dataSize(), snapshot.size());
+  EXPECT_EQ(std::vector<std::uint8_t>(event.frame.data(), event.frame.data() + event.frame.dataSize()), snapshot);
+}
+
+} // namespace livekit::test