pre-encoded ingest through ffi (#260)

Author: stephen-derosa

CMakeLists.txt

@@ -381,6 +381,7 @@ add_library(livekit SHARED
   src/data_track_schema.cpp
   src/data_track_stream.cpp
   src/e2ee.cpp
+  src/encoded_video_source.cpp
   src/ffi_handle.cpp
   src/ffi_client.cpp
   src/ffi_client.h

client-sdk-rust

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <optional>
+#include <vector>
+
+#include "livekit/video_codec.h"
+#include "livekit/video_source.h"
+#include "livekit/visibility.h"
+
+namespace livekit {
+
+/// @brief Video source for publishing pre-encoded access units without re-encoding.
+///
+/// Capture calls are synchronous and copy the payload before returning. This
+/// type is not safe for concurrent capture calls. Feedback polling can run on
+/// another application thread.
+class LIVEKIT_API EncodedVideoSource final : public VideoSource {
+public:
+  /// @brief One complete pre-encoded video access unit.
+  struct Frame {
+    /// True if this access unit is independently decodable.
+    bool is_keyframe = false;
+    /// Encoded frame width in pixels.
+    /// Set both width and height to zero to use the source resolution.
+    std::uint32_t width = 0;
+    /// Encoded frame height in pixels.
+    /// Set both width and height to zero to use the source resolution.
+    std::uint32_t height = 0;
+    /// Capture timestamp in microseconds. Set to zero to use the current time.
+    std::int64_t timestamp_us = 0;
+    /// Complete encoded access-unit payload.
+    std::vector<std::uint8_t> data;
+    /// Optional packet-trailer metadata.
+    std::optional<VideoFrameMetadata> metadata;
+  };
+
+  /// @brief Latest encoder rate-control target requested by the publishing pipeline.
+  struct RateControl {
+    /// Requested target bitrate in bits per second.
+    std::uint64_t target_bitrate_bps = 0;
+    /// Requested frame rate in frames per second.
+    double framerate_fps = 0.0;
+  };
+
+  /// @brief Pending feedback from the pre-encoded passthrough encoder.
+  struct Feedback {
+    /// True when the upstream encoder must produce a key frame.
+    bool keyframe_requested = false;
+    /// Latest rate-control target, if the publishing pipeline requested one.
+    std::optional<RateControl> rate_control;
+  };
+
+  /// @brief Create a pre-encoded source for one codec and initial resolution.
+  /// @param codec Codec carried by every frame submitted to this source. This
+  ///              must match TrackPublishOptions::video_codec.
+  /// @param width Initial source width in pixels. Must be in [1, 65535].
+  /// @param height Initial source height in pixels. Must be in [1, 65535].
+  /// @throws std::invalid_argument if either dimension is outside [1, 65535].
+  /// @throws std::runtime_error if source creation fails.
+  EncodedVideoSource(VideoCodec codec, int width, int height);
+
+  /// @brief Codec carried by every frame submitted to this source.
+  VideoCodec codec() const noexcept { return codec_; }
+
+  /// @brief Submit one complete encoded access unit.
+  /// @param frame Encoded frame. Its payload is copied during this call.
+  /// @return True if the frame was accepted.
+  /// @throws std::invalid_argument if the frame is empty, too large, has only
+  ///         one zero dimension, or has a dimension above 65535.
+  /// @throws std::runtime_error if the FFI request fails.
+  [[nodiscard]] bool captureFrame(const Frame& frame) const;
+
+  /// @brief Consume pending keyframe and rate-control feedback.
+  /// @return Feedback accumulated since the previous call. Rate-control
+  ///         updates use latest-value-wins semantics.
+  /// @throws std::runtime_error if the FFI request fails.
+  [[nodiscard]] Feedback takeFeedback() const;
+
+private:
+  using VideoSource::captureFrame;
+
+  VideoCodec codec_;
+};
+
+} // namespace livekit

include/livekit/encoded_video_source.h

@@ -22,6 +22,7 @@
 #include "livekit/audio_stream.h"
 #include "livekit/build.h"
 #include "livekit/e2ee.h"
+#include "livekit/encoded_video_source.h"
 #include "livekit/local_audio_track.h"
 #include "livekit/local_participant.h"
 #include "livekit/local_track_publication.h"
@@ -37,6 +38,7 @@
 #include "livekit/token_source.h"
 #include "livekit/tracing.h"
 #include "livekit/track_publication.h"
+#include "livekit/video_codec.h"
 #include "livekit/video_frame.h"
 #include "livekit/video_source.h"
 #include "livekit/video_stream.h"

include/livekit/livekit.h

@@ -146,6 +146,10 @@ class LIVEKIT_API LocalParticipant : public Participant {
   ///
   /// The caller retains ownership of @p source and should use it directly
   /// for frame capture on the video thread.
+  ///
+  /// If @p source is an @ref EncodedVideoSource, the track is published with
+  /// @ref VideoEncoderBackend::PreEncoded and the source's codec. Use
+  /// @ref publishTrack directly to control any other publish option.
   std::shared_ptr<LocalVideoTrack> publishVideoTrack(const std::string& name,
                                                      const std::shared_ptr<VideoSource>& source,
                                                      TrackSource track_source);

include/livekit/local_participant.h

@@ -23,6 +23,8 @@
 #include <string>
 #include <vector>
 
+#include "livekit/video_codec.h"
+
 namespace livekit {
 
 // Forward declarations to avoid pulling in heavy headers.
@@ -34,7 +36,6 @@ class LocalTrackPublication;
 class RemoteTrackPublication;
 class TrackPublication;
 
-enum class VideoCodec;
 enum class TrackSource;
 
 /// Overall quality of a participant's connection.
@@ -291,6 +292,24 @@ struct AudioEncodingOptions {
   std::uint64_t max_bitrate = 0;
 };
 
+/// @brief Preferred encoder backend for a published video track.
+enum class VideoEncoderBackend {
+  /// Use the SDK's default encoder selection.
+  Auto = 0,
+  /// Prefer a software encoder.
+  Software = 1,
+  /// Prefer any available hardware encoder.
+  Hardware = 2,
+  /// Prefer NVIDIA NVENC.
+  Nvenc = 3,
+  /// Prefer VAAPI.
+  Vaapi = 4,
+  /// Prefer VideoToolbox on Apple platforms.
+  VideoToolbox = 5,
+  /// Pass pre-encoded access units through without encoding them again.
+  PreEncoded = 6,
+};
+
 /// @brief Controls how the encoder degrades quality when bandwidth is constrained.
 enum class DegradationPreference {
   /// Balance between framerate and resolution degradation.
@@ -359,6 +378,10 @@ struct TrackPublishOptions {
   /// Controls how the encoder trades off between resolution and framerate
   /// when bandwidth is constrained. If not set, the server defaults apply.
   std::optional<DegradationPreference> degradation_preference;
+
+  /// Optional video encoder backend. Pre-encoded sources must select
+  /// @ref VideoEncoderBackend::PreEncoded.
+  std::optional<VideoEncoderBackend> video_encoder;
 };
 
 // ---------------------------------------------------------

include/livekit/room_event_types.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace livekit {
+
+/// @brief Codec used to publish a video track.
+enum class VideoCodec { VP8 = 0, H264 = 1, AV1 = 2, VP9 = 3, H265 = 4 };
+
+} // namespace livekit

include/livekit/video_codec.h

@@ -26,6 +26,7 @@
 namespace livekit {
 
 class VideoFrame;
+class EncodedVideoSource;
 
 /// Rotation of a video frame.
 ///
@@ -95,6 +96,11 @@ class LIVEKIT_API VideoSource {
                     VideoRotation rotation = VideoRotation::VIDEO_ROTATION_0);
 
 private:
+  friend class EncodedVideoSource;
+
+  enum class SourceType { Native, Encoded };
+  VideoSource(int width, int height, SourceType source_type);
+
   FfiHandle handle_; // owned FFI handle
   int width_{0};
   int height_{0};

include/livekit/video_source.h

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "livekit/encoded_video_source.h"
+
+#include <cstddef>
+#include <stdexcept>
+
+#include "ffi.pb.h"
+#include "ffi_client.h"
+#include "video_frame.pb.h"
+#include "video_utils.h"
+
+namespace livekit {
+namespace {
+
+constexpr std::size_t kMaxFrameSize = std::size_t{64} * 1024U * 1024U;
+constexpr std::uint32_t kMaxFrameDimension = 65535U;
+
+int validateDimension(int dimension) {
+  if (dimension <= 0 || static_cast<std::uint32_t>(dimension) > kMaxFrameDimension) {
+    throw std::invalid_argument("EncodedVideoSource: dimensions must be positive and at most 65535");
+  }
+  return dimension;
+}
+
+proto::VideoCodec toProtoCodec(VideoCodec codec) {
+  switch (codec) {
+    case VideoCodec::H264:
+      return proto::VideoCodec::H264;
+    case VideoCodec::H265:
+      return proto::VideoCodec::H265;
+    case VideoCodec::VP8:
+      return proto::VideoCodec::VP8;
+    case VideoCodec::VP9:
+      return proto::VideoCodec::VP9;
+    case VideoCodec::AV1:
+      return proto::VideoCodec::AV1;
+  }
+  throw std::invalid_argument("EncodedVideoSource: unknown codec");
+}
+
+} // namespace
+
+EncodedVideoSource::EncodedVideoSource(VideoCodec codec, int width, int height)
+    : VideoSource(validateDimension(width), validateDimension(height), VideoSource::SourceType::Encoded),
+      codec_(codec) {}
+
+bool EncodedVideoSource::captureFrame(const Frame& frame) const {
+  if ((frame.width == 0) != (frame.height == 0)) {
+    throw std::invalid_argument("EncodedVideoSource: frame dimensions must both be zero or both be non-zero");
+  }
+  if (frame.width > kMaxFrameDimension || frame.height > kMaxFrameDimension) {
+    throw std::invalid_argument("EncodedVideoSource: frame dimensions must not exceed 65535");
+  }
+  if (frame.data.empty() || frame.data.size() > kMaxFrameSize) {
+    throw std::invalid_argument("EncodedVideoSource: frame payload must be between 1 byte and 64 MiB");
+  }
+  if (ffiHandleId() == 0) {
+    throw std::runtime_error("EncodedVideoSource: invalid FFI handle");
+  }
+
+  proto::FfiRequest req;
+  auto* msg = req.mutable_capture_encoded_video_frame();
+  msg->set_source_handle(ffiHandleId());
+  auto* buffer = msg->mutable_buffer();
+  buffer->set_data_ptr(reinterpret_cast<std::uintptr_t>(frame.data.data()));
+  buffer->set_data_len(frame.data.size());
+  msg->set_codec(toProtoCodec(codec_));
+  msg->set_frame_type(frame.is_keyframe ? proto::EncodedFrameType::ENCODED_FRAME_KEY
+                                        : proto::EncodedFrameType::ENCODED_FRAME_DELTA);
+  msg->set_width(frame.width == 0 ? static_cast<std::uint32_t>(width()) : frame.width);
+  msg->set_height(frame.height == 0 ? static_cast<std::uint32_t>(height()) : frame.height);
+  msg->set_timestamp_us(frame.timestamp_us);
+  if (auto metadata = toProto(frame.metadata)) {
+    msg->mutable_metadata()->CopyFrom(*metadata);
+  }
+
+  const proto::FfiResponse resp = FfiClient::instance().sendRequest(req);
+  if (!resp.has_capture_encoded_video_frame()) {
+    throw std::runtime_error("EncodedVideoSource: missing capture response");
+  }
+  return resp.capture_encoded_video_frame().accepted();
+}
+
+EncodedVideoSource::Feedback EncodedVideoSource::takeFeedback() const {
+  if (ffiHandleId() == 0) {
+    throw std::runtime_error("EncodedVideoSource: invalid FFI handle");
+  }
+
+  proto::FfiRequest req;
+  req.mutable_take_encoded_video_source_feedback()->set_source_handle(ffiHandleId());
+  const proto::FfiResponse resp = FfiClient::instance().sendRequest(req);
+  if (!resp.has_take_encoded_video_source_feedback()) {
+    throw std::runtime_error("EncodedVideoSource: missing feedback response");
+  }
+
+  const auto& proto_feedback = resp.take_encoded_video_source_feedback();
+  Feedback feedback;
+  feedback.keyframe_requested = proto_feedback.keyframe_requested();
+  if (proto_feedback.has_rate_control()) {
+    feedback.rate_control =
+        RateControl{proto_feedback.rate_control().target_bitrate_bps(), proto_feedback.rate_control().framerate_fps()};
+  }
+  return feedback;
+}
+
+} // namespace livekit

src/encoded_video_source.cpp

@@ -17,11 +17,13 @@
 #include "livekit/local_participant.h"
 
 #include <chrono>
+#include <memory>
 #include <stdexcept>
 
 #include "data_track.pb.h"
 #include "ffi.pb.h"
 #include "ffi_client.h"
+#include "livekit/encoded_video_source.h"
 #include "livekit/ffi_handle.h"
 #include "livekit/local_audio_track.h"
 #include "livekit/local_data_track.h"
@@ -209,6 +211,14 @@ std::shared_ptr<LocalVideoTrack> LocalParticipant::publishVideoTrack(const std::
   auto track = LocalVideoTrack::createLocalVideoTrack(name, source);
   TrackPublishOptions opts;
   opts.source = track_source;
+  // A pre-encoded source cannot be re-encoded, so the passthrough backend and
+  // the source's codec are the only valid choices here.
+  if (const auto encoded = std::dynamic_pointer_cast<EncodedVideoSource>(source)) {
+    opts.video_encoder = VideoEncoderBackend::PreEncoded;
+    opts.video_codec = encoded->codec();
+    // A single-stream encoded source cannot supply the requested layers, so turn off simulcast.
+    opts.simulcast = false;
+  }
   publishTrack(track, opts);
   return track;
 }