feat: stream a whole monitor and leave out the caller's own windows

Author: developer239

README.md

@@ -17,7 +17,7 @@ robot-cpp is built around the distinctions those libraries collapse:
 
 ## What it does
 
-- Move the cursor (absolute or smoothly interpolated), click, double-click, drag, and scroll, with buttons through X1/X2 and line- or pixel-unit scrolling.
+- Move the cursor (absolute, smoothly interpolated, or by a raw relative delta for software that captures it), click, double-click, drag, and scroll, with buttons through X1/X2 and line- or pixel-unit scrolling.
 - Press and release physical keys by position, build modifier chords, and type arbitrary Unicode text independent of the active keyboard layout.
 - Enumerate every display with its own scale factor and both logical and physical bounds, capture any region or a whole monitor at native pixel resolution, sample individual pixels, and encode captures as PNG.
 - Enumerate on-screen application windows, query their current bounds and z-order, activate their owning application, and stream a single window as zero-copy video frames carrying the density and content-rect metadata needed to map frame pixels back to desktop coordinates (macOS ScreenCaptureKit; reported as Unsupported elsewhere until those backends exist).
@@ -159,6 +159,7 @@ Mouse control operates in global logical coordinates (the virtual desktop shared
 ```cpp
 mouse.move({800.0, 450.0});          // absolute warp in logical coordinates
 mouse.moveSmooth({100.0, 100.0});    // interpolated over a duration
+mouse.moveBy(12.0, -7.0);            // raw delta, for game cameras and mouse steering
 
 mouse.click();                       // left click at the current position
 mouse.click(robot::MouseButton::Right);
@@ -238,6 +239,8 @@ if (auto point = robot::toWindowPoint(frame.info(), pixelX, pixelY)) {
 }
 ```
 
+`windows.streamMonitor(monitor.id, options, onFrame, onError)` streams a whole monitor under the same rules (macOS ScreenCaptureKit; Unsupported elsewhere until those backends exist). By default it leaves the calling process's own windows out of the frames, so an overlay never captures itself; set `excludesOwnWindows` to false to include them.
+
 `topmostWindowId()` reports the topmost normal-layer window in z-order (an ordering query - the honest primitive for "is my target frontmost"), and `activateApplicationFor(windowId)` brings a window's owning application forward; which of its windows ends up on top is the OS's decision.
 
 ## Recording and replay

include/robot/WindowStream.h

@@ -14,6 +14,9 @@ class IWindowStreamBackend;
 struct WindowStreamOptions {
   double maxFps = 60.0;  // must be finite and in [1, 1000]
   bool showsCursor = true;
+  // Monitor streams only: leave this process's own windows out of the frames,
+  // so an overlay never sees itself. A window stream only ever shows its window.
+  bool excludesOwnWindows = true;
 };
 
 // Invoked on the stream's internal capture thread for every delivered frame,

include/robot/Windows.h

@@ -60,7 +60,18 @@ class Windows {
       VideoFrameCallback onFrame, StreamErrorCallback onError = nullptr
   );
 
+  // Starts streaming a whole monitor, by Monitor::id. For an overlay, or for
+  // following anything on the desktop rather than one window.
+  [[nodiscard]] std::expected<WindowStream, Error> streamMonitor(
+      std::uint32_t monitorId, const WindowStreamOptions& options,
+      VideoFrameCallback onFrame, StreamErrorCallback onError = nullptr
+  );
+
  private:
+  [[nodiscard]] std::expected<void, Error> validateStream(
+      const WindowStreamOptions& options, const VideoFrameCallback& onFrame
+  ) const;
+
   backend::IWindowBackend* backend_;
 };
 

include/robot/backend/IWindowBackend.h

@@ -68,6 +68,15 @@ class IWindowBackend {
       std::uint32_t windowId, const WindowStreamOptions& options,
       VideoFrameCallback onFrame, StreamErrorCallback onError
   ) = 0;
+
+  // Starts streaming a whole monitor, under the same delivery rules. Frames
+  // report windowId 0.
+  [[nodiscard]] virtual std::expected<
+      std::unique_ptr<IWindowStreamBackend>, Error>
+  startMonitorStream(
+      std::uint32_t monitorId, const WindowStreamOptions& options,
+      VideoFrameCallback onFrame, StreamErrorCallback onError
+  ) = 0;
 };
 
 }  // namespace robot::backend

src/common/Windows.cpp

@@ -61,14 +61,13 @@ std::expected<void, Error> Windows::activateApplicationFor(
   return backend_->activateApplicationForWindow(windowId);
 }
 
-std::expected<WindowStream, Error> Windows::stream(
-    const std::uint32_t windowId, const WindowStreamOptions& options,
-    VideoFrameCallback onFrame, StreamErrorCallback onError
-) {
+std::expected<void, Error> Windows::validateStream(
+    const WindowStreamOptions& options, const VideoFrameCallback& onFrame
+) const {
   if (backend_ == nullptr) return std::unexpected(Error::unsupported(kNoBackend));
   if (onFrame == nullptr) {
     return std::unexpected(
-        Error::invalidArgument("stream() requires a frame callback")
+        Error::invalidArgument("streaming requires a frame callback")
     );
   }
   if (!std::isfinite(options.maxFps) || options.maxFps < 1.0 ||
@@ -78,11 +77,37 @@ std::expected<WindowStream, Error> Windows::stream(
     )));
   }
 
+  return {};
+}
+
+std::expected<WindowStream, Error> Windows::stream(
+    const std::uint32_t windowId, const WindowStreamOptions& options,
+    VideoFrameCallback onFrame, StreamErrorCallback onError
+) {
+  if (auto valid = validateStream(options, onFrame); !valid) {
+    return std::unexpected(valid.error());
+  }
+
   auto stream = backend_->startStream(
       windowId, options, std::move(onFrame), std::move(onError)
   );
   if (!stream) return std::unexpected(stream.error());
   return WindowStream(std::move(*stream));
 }
 
+std::expected<WindowStream, Error> Windows::streamMonitor(
+    const std::uint32_t monitorId, const WindowStreamOptions& options,
+    VideoFrameCallback onFrame, StreamErrorCallback onError
+) {
+  if (auto valid = validateStream(options, onFrame); !valid) {
+    return std::unexpected(valid.error());
+  }
+
+  auto stream = backend_->startMonitorStream(
+      monitorId, options, std::move(onFrame), std::move(onError)
+  );
+  if (!stream) return std::unexpected(stream.error());
+  return WindowStream(std::move(*stream));
+}
+
 }  // namespace robot

src/platform/macos/MacWindowBackend.h

@@ -32,6 +32,12 @@ class MacWindowBackend final : public backend::IWindowBackend {
       std::uint32_t windowId, const WindowStreamOptions& options,
       VideoFrameCallback onFrame, StreamErrorCallback onError
   ) override;
+
+  [[nodiscard]] std::expected<std::unique_ptr<backend::IWindowStreamBackend>, Error>
+  startMonitorStream(
+      std::uint32_t monitorId, const WindowStreamOptions& options,
+      VideoFrameCallback onFrame, StreamErrorCallback onError
+  ) override;
 };
 
 }  // namespace robot::mac

src/platform/macos/MacWindowBackend.mm

@@ -6,9 +6,11 @@
 #import <ScreenCaptureKit/ScreenCaptureKit.h>
 
 #include <mach/mach_time.h>
+#include <unistd.h>
 
 #include <atomic>
 #include <chrono>
+#include <format>
 #include <memory>
 #include <optional>
 
@@ -295,6 +297,79 @@ void stop() override {
   std::shared_ptr<StreamChannel> channel_;
 };
 
+// Starts capture of whatever the filter selects, at its native pixel size.
+std::expected<std::unique_ptr<backend::IWindowStreamBackend>, Error>
+startFilteredStream(
+    SCContentFilter* filter, const std::uint32_t windowId,
+    const WindowStreamOptions& options, VideoFrameCallback onFrame,
+    StreamErrorCallback onError
+) {
+  SCStreamConfiguration* config = [[SCStreamConfiguration alloc] init];
+  const double scale = static_cast<double>(filter.pointPixelScale);
+  config.width = static_cast<size_t>(filter.contentRect.size.width * scale);
+  config.height = static_cast<size_t>(filter.contentRect.size.height * scale);
+  config.pixelFormat = kCVPixelFormatType_32BGRA;
+  config.minimumFrameInterval =
+      CMTimeMake(1, static_cast<std::int32_t>(options.maxFps));
+  config.queueDepth = 3;
+  config.showsCursor = options.showsCursor ? YES : NO;
+
+  dispatch_queue_t queue =
+      dispatch_queue_create("robot.window-stream", DISPATCH_QUEUE_SERIAL);
+  auto channel = std::make_shared<StreamChannel>();
+  channel->onFrame = std::move(onFrame);
+  channel->onError = std::move(onError);
+  channel->queue = queue;
+  channel->windowId = windowId;
+  channel->active.store(true);
+
+  RobotWindowStreamOutput* output = [[RobotWindowStreamOutput alloc] init];
+  output->channel = channel;
+  SCStream* stream = [[SCStream alloc] initWithFilter:filter
+                                        configuration:config
+                                             delegate:output];
+
+  NSError* addError = nil;
+  [stream addStreamOutput:output
+                     type:SCStreamOutputTypeScreen
+       sampleHandlerQueue:queue
+                    error:&addError];
+  if (addError != nil) {
+    return std::unexpected(
+        Error::captureFailed(addError.localizedDescription.UTF8String)
+    );
+  }
+
+  dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
+  __block NSError* startError = nil;
+  [stream startCaptureWithCompletionHandler:^(NSError* err) {
+    startError = err;
+    dispatch_semaphore_signal(semaphore);
+  }];
+  const auto deadline =
+      dispatch_time(DISPATCH_TIME_NOW, 5 * static_cast<int64_t>(NSEC_PER_SEC));
+  if (dispatch_semaphore_wait(semaphore, deadline) != 0) {
+    // Startup may still complete later; deactivate the channel and issue a
+    // fire-and-forget stop so a late success never delivers frames.
+    // startError is deliberately not read here - the completion handler may
+    // still be about to write it.
+    channel->active.store(false);
+    [stream stopCaptureWithCompletionHandler:^(NSError*) {
+    }];
+    return std::unexpected(Error::captureFailed("timed out starting capture"));
+  }
+  if (startError != nil) {
+    channel->active.store(false);
+    return std::unexpected(
+        Error::captureFailed(startError.localizedDescription.UTF8String)
+    );
+  }
+
+  return std::make_unique<MacWindowStream>(
+      stream, output, queue, std::move(channel)
+  );
+}
+
 }  // namespace
 
 std::expected<std::vector<Window>, Error> MacWindowBackend::enumerateWindows() {
@@ -444,69 +519,55 @@ void stop() override {
 
   SCContentFilter* filter =
       [[SCContentFilter alloc] initWithDesktopIndependentWindow:scWindow];
-  SCStreamConfiguration* config = [[SCStreamConfiguration alloc] init];
-  const double scale = static_cast<double>(filter.pointPixelScale);
-  config.width = static_cast<size_t>(filter.contentRect.size.width * scale);
-  config.height = static_cast<size_t>(filter.contentRect.size.height * scale);
-  config.pixelFormat = kCVPixelFormatType_32BGRA;
-  config.minimumFrameInterval =
-      CMTimeMake(1, static_cast<std::int32_t>(options.maxFps));
-  config.queueDepth = 3;
-  config.showsCursor = options.showsCursor ? YES : NO;
-
-  dispatch_queue_t queue =
-      dispatch_queue_create("robot.window-stream", DISPATCH_QUEUE_SERIAL);
-  auto channel = std::make_shared<StreamChannel>();
-  channel->onFrame = std::move(onFrame);
-  channel->onError = std::move(onError);
-  channel->queue = queue;
-  channel->windowId = windowId;
-  channel->active.store(true);
 
-  RobotWindowStreamOutput* output = [[RobotWindowStreamOutput alloc] init];
-  output->channel = channel;
-  SCStream* stream = [[SCStream alloc] initWithFilter:filter
-                                        configuration:config
-                                             delegate:output];
+  return startFilteredStream(
+      filter, windowId, options, std::move(onFrame), std::move(onError)
+  );
+}
 
-  NSError* addError = nil;
-  [stream addStreamOutput:output
-                     type:SCStreamOutputTypeScreen
-       sampleHandlerQueue:queue
-                    error:&addError];
-  if (addError != nil) {
+std::expected<std::unique_ptr<backend::IWindowStreamBackend>, Error>
+MacWindowBackend::startMonitorStream(
+    const std::uint32_t monitorId, const WindowStreamOptions& options,
+    VideoFrameCallback onFrame, StreamErrorCallback onError
+) {
+  if (@available(macOS 14.0, *)) {
+    // Streaming is available; continue below.
+  } else {
     return std::unexpected(
-        Error::captureFailed(addError.localizedDescription.UTF8String)
+        Error::unsupported("monitor streaming requires macOS 14 or newer")
     );
   }
 
-  dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
-  __block NSError* startError = nil;
-  [stream startCaptureWithCompletionHandler:^(NSError* err) {
-    startError = err;
-    dispatch_semaphore_signal(semaphore);
-  }];
-  const auto deadline =
-      dispatch_time(DISPATCH_TIME_NOW, 5 * static_cast<int64_t>(NSEC_PER_SEC));
-  if (dispatch_semaphore_wait(semaphore, deadline) != 0) {
-    // Startup may still complete later; deactivate the channel and issue a
-    // fire-and-forget stop so a late success never delivers frames.
-    // startError is deliberately not read here - the completion handler may
-    // still be about to write it.
-    channel->active.store(false);
-    [stream stopCaptureWithCompletionHandler:^(NSError*) {
-    }];
-    return std::unexpected(Error::captureFailed("timed out starting capture"));
+  auto contentResult = fetchShareableContent();
+  if (!contentResult) return std::unexpected(contentResult.error());
+
+  SCDisplay* scDisplay = nil;
+  for (SCDisplay* display in (*contentResult).displays) {
+    if (display.displayID == monitorId) {
+      scDisplay = display;
+      break;
+    }
   }
-  if (startError != nil) {
-    channel->active.store(false);
-    return std::unexpected(
-        Error::captureFailed(startError.localizedDescription.UTF8String)
-    );
+  if (scDisplay == nil) {
+    return std::unexpected(Error::invalidArgument(
+        std::format("no monitor with id {}", monitorId)
+    ));
   }
 
-  return std::make_unique<MacWindowStream>(
-      stream, output, queue, std::move(channel)
+  NSMutableArray<SCRunningApplication*>* excluded = [NSMutableArray array];
+  if (options.excludesOwnWindows) {
+    for (SCRunningApplication* app in (*contentResult).applications) {
+      if (app.processID == getpid()) [excluded addObject:app];
+    }
+  }
+
+  SCContentFilter* filter =
+      [[SCContentFilter alloc] initWithDisplay:scDisplay
+                         excludingApplications:excluded
+                              exceptingWindows:@[]];
+
+  return startFilteredStream(
+      filter, 0, options, std::move(onFrame), std::move(onError)
   );
 }