feat: capture windows and monitors on Windows with Windows Graphics Capture

Author: developer239

CMakeLists.txt

@@ -94,6 +94,7 @@ elseif(WIN32)
         src/platform/windows/WinMouseBackend.cpp
         src/platform/windows/WinScreenBackend.cpp
         src/platform/windows/WinEventTapBackend.cpp
+        src/platform/windows/WinWindowBackend.cpp
     )
 elseif(UNIX AND NOT APPLE)
     set(ROBOT_PLATFORM_SOURCES
@@ -166,7 +167,8 @@ elseif(WIN32)
     # user32/gdi32 for SendInput, hooks, GDI capture. Shcore and the DPI context
     # API are resolved at runtime (see WinDpi.cpp / WinScreenBackend.cpp), so they
     # are not linked here.
-    target_link_libraries(robot PRIVATE user32 gdi32)
+    # d3d11, dxgi, dwmapi and windowsapp for Windows Graphics Capture.
+    target_link_libraries(robot PRIVATE user32 gdi32 d3d11 dxgi dwmapi shcore windowsapp)
     target_compile_definitions(robot PRIVATE
         WIN32_LEAN_AND_MEAN NOMINMAX UNICODE _UNICODE)
 elseif(UNIX AND NOT APPLE)

include/robot/win/VideoFrame.h

@@ -0,0 +1,25 @@
+#pragma once
+
+// Typed Windows bridge for VideoFrame's platform buffer. Include this from
+// Windows-only translation units instead of casting nativeHandle() by hand.
+
+#include <cstdint>
+
+#include "robot/VideoFrame.h"
+
+namespace robot::win {
+
+// Frame pixels mapped for CPU reads, BGRA and top-down. Valid for the frame's
+// lifetime, and safe to read from any thread.
+struct PixelBuffer {
+  const std::uint8_t* pixels = nullptr;
+  int width = 0;
+  int height = 0;
+  int bytesPerRow = 0;
+};
+
+[[nodiscard]] inline const PixelBuffer& pixelBuffer(const VideoFrame& frame) {
+  return *static_cast<const PixelBuffer*>(frame.nativeHandle());
+}
+
+}  // namespace robot::win

src/platform/windows/WinBackendFactory.cpp

@@ -24,6 +24,8 @@ Capabilities buildCapabilities() {
   c.canCaptureScreen = true;
   c.canEnumerateMonitors = true;
   c.canRecordEvents = true;
+  c.canEnumerateWindows = true;
+  c.canStreamWindows = true;
 
   c.requiresAccessibilityPermission = false;
   c.requiresScreenRecordingPermission = false;

src/platform/windows/WinPlatformBackend.h

@@ -6,11 +6,12 @@
 #include "WinKeyboardBackend.h"
 #include "WinMouseBackend.h"
 #include "WinScreenBackend.h"
+#include "WinWindowBackend.h"
 #include "robot/backend/IPlatformBackend.h"
 
 namespace robot::win {
 
-// Owns the four Windows sub-backends and the capabilities report. A Session owns
+// Owns the five Windows sub-backends and the capabilities report. A Session owns
 // exactly one of these.
 class WinPlatformBackend final : public backend::IPlatformBackend {
  public:
@@ -19,14 +20,14 @@ class WinPlatformBackend final : public backend::IPlatformBackend {
         keyboard_(std::make_unique<WinKeyboardBackend>()),
         mouse_(std::make_unique<WinMouseBackend>()),
         screen_(std::make_unique<WinScreenBackend>()),
-        eventTap_(std::make_unique<WinEventTapBackend>()) {}
+        eventTap_(std::make_unique<WinEventTapBackend>()),
+        windows_(std::make_unique<WinWindowBackend>()) {}
 
   backend::IKeyboardBackend& keyboard() override { return *keyboard_; }
   backend::IMouseBackend& mouse() override { return *mouse_; }
   backend::IScreenBackend& screen() override { return *screen_; }
   backend::IEventTapBackend* eventTap() override { return eventTap_.get(); }
-  // Window enumeration/capture is not implemented on this platform yet.
-  backend::IWindowBackend* windows() override { return nullptr; }
+  backend::IWindowBackend* windows() override { return windows_.get(); }
   const Capabilities& capabilities() const override { return capabilities_; }
 
  private:
@@ -35,6 +36,7 @@ class WinPlatformBackend final : public backend::IPlatformBackend {
   std::unique_ptr<WinMouseBackend> mouse_;
   std::unique_ptr<WinScreenBackend> screen_;
   std::unique_ptr<WinEventTapBackend> eventTap_;
+  std::unique_ptr<WinWindowBackend> windows_;
 };
 
 }  // namespace robot::win