feat: report where the user is typing, so a client can keep it on screen

A phone's on-screen keyboard covers half the picture, and the client cannot tell
which half matters — the field being typed into is usually the hidden one. The
host can tell, so GET /caret reports it.

The answer is fractions of the streamed display, so the client needs to know
nothing about resolutions, and "source" says which of two things it is:

- "caret", the focused application's insertion point, read through Accessibility.
  macOS only, because no other platform exposes one.
- "pointer", where the cursor is. The fallback for the applications that report
  no caret, which is most of them, and a good stand-in: you click into a field to
  type in it. It is also the only thing that helps in trackpad mode, where the
  client sends relative motion and never learns where the pointer ended up.

An empty object means neither was available and the client should leave the
picture where it is.

Served over HTTPS only, so it reaches paired and enabled clients alone. Where
someone is typing, and the pointer position it falls back to, describe what the
user is doing closely enough to belong behind the same verification as the rest
of the session.

platf::pointer_location() is declared in platform/common.h and implemented on
each platform. get_mouse_loc() was the obvious thing to reuse and is not usable
here: it is documented as existing only for tests, and it takes the input backend
of a running stream, which an HTTP handler does not have.

Windows reads GetCursorPos and normalises against the primary monitor. Linux
queries X11 and normalises against the X screen, which is what x11grab captures;
Wayland offers no way to ask, so it reports nothing.

Three details on the macOS side, each found by probing a machine rather than
from documentation:

- Ask the application, not the system. AXUIElementCreateSystemWide() with
  kAXFocusedUIElementAttribute returns nothing for applications that answer when
  asked directly, iTerm2 among them.
- Filter by display. Accessibility works in whole-desktop coordinates, so a
  display above the main one gives negative values, and a caret on a display that
  is not being streamed means nothing to the client.
- An empty rect at the origin means no caret. Elements without an insertion point
  return that rather than an error, and taking it at face value puts the caret in
  the top-left corner.

Verified on macOS 26.6.1: iTerm2 and Xcode report a caret; Brave, VS Code,
Sublime Text and Telegram do not and fall back to the pointer.

Author: harrison001

src/nvhttp.cpp

@@ -28,6 +28,9 @@
 #include "logging.h"
 #include "network.h"
 #include "nvhttp.h"
+#ifdef __APPLE__
+  #include "src/platform/macos/misc.h"
+#endif
 #include "platform/common.h"
 #include "process.h"
 #include "rtsp.h"
@@ -981,6 +984,57 @@ namespace nvhttp {
     }
   }
 
+  /**
+   * @brief Report where the focused application is expecting text.
+   *
+   * A phone's on-screen keyboard covers half the picture, and the client has no way of knowing
+   * which half matters. The host does. Coordinates are fractions of the streamed display so the
+   * client needs to know nothing about resolutions, and "source" says whether the answer is the
+   * insertion point itself or the pointer standing in for it. An empty body means neither was
+   * available, and the client should leave the picture where it is.
+   *
+   * Served only over HTTPS, so it reaches paired and enabled clients alone. Where someone is
+   * typing, and the pointer position it falls back to, describe what the user is doing closely
+   * enough that they belong behind the same verification as the rest of the session.
+   *
+   * @param response HTTP response object to populate.
+   * @param request HTTP request data from the client.
+   */
+  void caret(resp_https_t response, req_https_t request) {
+    print_req<SunshineHTTPS>(request);
+
+    SimpleWeb::CaseInsensitiveMultimap headers;
+    headers.emplace("Content-Type", "application/json");
+
+    // The caret when the focused application will say where it is, the pointer when it will not,
+    // which is most of them. Accessibility is the only interface that reports an insertion point
+    // and it is macOS-only, so elsewhere the pointer is the whole answer.
+#ifdef __APPLE__
+    if (const auto rect = platf::focused_caret()) {
+      const auto body = "{\"x\":" + std::to_string((*rect)[0]) +
+                        ",\"y\":" + std::to_string((*rect)[1]) +
+                        ",\"w\":" + std::to_string((*rect)[2]) +
+                        ",\"h\":" + std::to_string((*rect)[3]) +
+                        ",\"source\":\"caret\"}";
+      response->write(SimpleWeb::StatusCode::success_ok, body, headers);
+      return;
+    }
+#endif
+
+    // Where you clicked to start typing, so it is close enough to the field to be worth moving
+    // the picture for, and in trackpad mode it is the only thing the client cannot work out for
+    // itself: it sends relative motion and never learns where the pointer ended up.
+    if (const auto point = platf::pointer_location()) {
+      const auto body = "{\"x\":" + std::to_string((*point)[0]) +
+                        ",\"y\":" + std::to_string((*point)[1]) +
+                        ",\"w\":0,\"h\":0,\"source\":\"pointer\"}";
+      response->write(SimpleWeb::StatusCode::success_ok, body, headers);
+      return;
+    }
+
+    response->write(SimpleWeb::StatusCode::success_ok, "{}", headers);
+  }
+
   /**
    * @brief Launch the requested application for a GameStream session.
    *
@@ -1362,6 +1416,7 @@ namespace nvhttp {
       pair<SunshineHTTPS>(add_cert, resp, req);
     };
     https_server.resource["^/applist$"]["GET"] = applist;
+    https_server.resource["^/caret$"]["GET"] = caret;
     https_server.resource["^/appasset$"]["GET"] = appasset;
     https_server.resource["^/launch$"]["GET"] = [&host_audio](auto resp, auto req) {
       launch(host_audio, resp, req);

src/platform/common.h

@@ -5,6 +5,7 @@
 #pragma once
 
 // standard includes
+#include <array>
 #include <bitset>
 #include <filesystem>
 #include <functional>
@@ -1123,6 +1124,23 @@ namespace platf {
    * @examples_end
    */
   std::optional<util::point_t> get_mouse_loc(input_t &input);
+
+  /**
+   * @brief Where the pointer is, as a fraction of the display being streamed.
+   *
+   * Unlike `get_mouse_loc()` this takes no input backend and is meant to be read outside a
+   * session, and it answers in fractions rather than screen coordinates so a caller can use it
+   * without knowing the host's resolution.
+   *
+   * Every implementation measures against the primary display rather than resolving the one a
+   * session was configured to capture. That is the default in every case, and a pointer on any
+   * other display reports nothing rather than a number the client would misplace.
+   *
+   * @return `{x, y}` in 0..1, or `std::nullopt` when the pointer is on another display or the
+   *         platform cannot observe it.
+   */
+  std::optional<std::array<double, 2>> pointer_location();
+
   /**
    * @brief Move mouse using the backend coordinate system.
    *

src/platform/linux/input/virtualhid.cpp

@@ -66,6 +66,47 @@ namespace platf {
 #endif
   }
 
+  // Kept beside get_mouse_loc() rather than in misc.cpp, which is where the other two platforms
+  // put it, because this is the only Linux translation unit with an X11 connection already set up.
+  std::optional<std::array<double, 2>> pointer_location() {
+#ifdef SUNSHINE_BUILD_X11
+    auto *display = XOpenDisplay(nullptr);
+    if (!display) {
+      return std::nullopt;
+    }
+
+    const auto screen = DefaultScreen(display);
+    const auto width = static_cast<double>(DisplayWidth(display, screen));
+    const auto height = static_cast<double>(DisplayHeight(display, screen));
+
+    const auto root = DefaultRootWindow(display);
+    Window root_return {};
+    Window child_return {};
+    int root_x = 0;
+    int root_y = 0;
+    int window_x = 0;
+    int window_y = 0;
+    unsigned int mask = 0;
+    const auto queried = XQueryPointer(display, root, &root_return, &child_return, &root_x, &root_y, &window_x, &window_y, &mask);
+    XCloseDisplay(display);
+
+    if (!queried || width <= 0 || height <= 0) {
+      return std::nullopt;
+    }
+
+    // The X screen, which spans every output. That is also what x11grab captures, so the two
+    // agree by default; a session capturing one output of several would need the pointer placed
+    // against that output instead.
+    return std::array<double, 2> {
+      root_x / width,
+      root_y / height
+    };
+#else
+    // Wayland gives no way to ask, so the client falls back to leaving the picture alone.
+    return std::nullopt;
+#endif
+  }
+
   std::vector<supported_gamepad_t> &supported_gamepads(input_t *input) {
     static std::vector<supported_gamepad_t> gamepads;
     if (!input || !input->get()) {

src/platform/macos/misc.h

@@ -5,6 +5,8 @@
 #pragma once
 
 // standard includes
+#include <array>
+#include <optional>
 #include <vector>
 
 // platform includes
@@ -17,6 +19,21 @@ namespace platf {
    * @return True when Sunshine can capture the screen.
    */
   bool is_screen_capture_allowed();
+
+  /**
+   * @brief Where the focused application is expecting text, as a fraction of the streamed display.
+   *
+   * A client whose on-screen keyboard covers half the picture has no way of knowing which half
+   * matters. The host does: the focused element knows where its insertion point is, and
+   * Accessibility will say so. Normalised to 0..1 of the display so the client needs to know
+   * nothing about resolutions.
+   *
+   * Empty when the focused application does not report an insertion point — which is most of
+   * them — or when the caret is on a display other than the one being streamed.
+   *
+   * @return {x, y, width, height} in 0..1 of the streamed display, or nothing.
+   */
+  std::optional<std::array<double, 4>> focused_caret();
 }  // namespace platf
 
 namespace dyn {

src/platform/macos/misc.mm

@@ -19,6 +19,8 @@
 // platform includes
 #include <arpa/inet.h>
 #include <dlfcn.h>
+#include <ApplicationServices/ApplicationServices.h>
+#include <AppKit/AppKit.h>
 #include <Foundation/Foundation.h>
 #include <mach-o/dyld.h>
 #include <net/if_dl.h>
@@ -32,6 +34,7 @@
 
 // local includes
 #include "misc.h"
+#include "src/utility.h"
 #include "src/entry_handler.h"
 #include "src/logging.h"
 #include "src/platform/common.h"
@@ -69,6 +72,119 @@
   /**
    * @brief Check whether screen capture allowed.
    */
+  namespace {
+    /// Reads an Accessibility attribute, returning nothing rather than an error code.
+    CFTypeRef copy_attribute(AXUIElementRef element, CFStringRef name) {
+      CFTypeRef value = nullptr;
+      if (AXUIElementCopyAttributeValue(element, name, &value) != kAXErrorSuccess) {
+        return nullptr;
+      }
+      return value;
+    }
+  }  // namespace
+
+  std::optional<std::array<double, 4>> focused_caret() {
+    if (!AXIsProcessTrusted()) {
+      return std::nullopt;
+    }
+
+    NSRunningApplication *front = [[NSWorkspace sharedWorkspace] frontmostApplication];
+    if (front == nil) {
+      return std::nullopt;
+    }
+
+    // Ask the application, not the system. AXUIElementCreateSystemWide() with
+    // kAXFocusedUIElementAttribute comes back empty for applications that answer perfectly well
+    // when asked directly — iTerm2 among them, which is the case this exists for.
+    AXUIElementRef app = AXUIElementCreateApplication(front.processIdentifier);
+    if (!app) {
+      return std::nullopt;
+    }
+    // A busy or hung application must not stall the request that asked for this.
+    AXUIElementSetMessagingTimeout(app, 0.1f);
+
+    const auto release_app = util::fail_guard([app]() {
+      CFRelease(app);
+    });
+
+    CFTypeRef focused = copy_attribute(app, kAXFocusedUIElementAttribute);
+    if (!focused) {
+      return std::nullopt;
+    }
+    const auto release_focused = util::fail_guard([focused]() {
+      CFRelease(focused);
+    });
+
+    CFTypeRef range = copy_attribute(static_cast<AXUIElementRef>(focused), kAXSelectedTextRangeAttribute);
+    if (!range) {
+      return std::nullopt;
+    }
+    const auto release_range = util::fail_guard([range]() {
+      CFRelease(range);
+    });
+
+    CFTypeRef bounds = nullptr;
+    if (AXUIElementCopyParameterizedAttributeValue(
+          static_cast<AXUIElementRef>(focused),
+          kAXBoundsForRangeParameterizedAttribute,
+          range,
+          &bounds
+        ) != kAXErrorSuccess ||
+        !bounds) {
+      return std::nullopt;
+    }
+    const auto release_bounds = util::fail_guard([bounds]() {
+      CFRelease(bounds);
+    });
+
+    CGRect caret = CGRectZero;
+    if (!AXValueGetValue(static_cast<AXValueRef>(bounds), kAXValueTypeCGRect, &caret)) {
+      return std::nullopt;
+    }
+    // An element that does not really have an insertion point answers with an empty rect at the
+    // origin rather than with an error.
+    if (caret.size.width == 0 && caret.size.height == 0) {
+      return std::nullopt;
+    }
+
+    // Accessibility works in the coordinates of the whole desktop arrangement, which on a second
+    // display can be negative or larger than the streamed display. Only a caret on the display
+    // being streamed means anything to the client.
+    const CGRect display = CGDisplayBounds(CGMainDisplayID());
+    const CGPoint anchor = CGPointMake(CGRectGetMidX(caret), CGRectGetMidY(caret));
+    if (!CGRectContainsPoint(display, anchor)) {
+      return std::nullopt;
+    }
+
+    return std::array<double, 4> {
+      (caret.origin.x - display.origin.x) / display.size.width,
+      (caret.origin.y - display.origin.y) / display.size.height,
+      caret.size.width / display.size.width,
+      caret.size.height / display.size.height
+    };
+  }
+
+  std::optional<std::array<double, 2>> pointer_location() {
+    // A fresh event every time rather than a reused one, as get_mouse_loc() does: the location
+    // on a reused event is whatever it was when the event was made.
+    CGEventRef snapshot = CGEventCreate(nullptr);
+    if (!snapshot) {
+      return std::nullopt;
+    }
+    const CGPoint location = CGEventGetLocation(snapshot);
+    CFRelease(snapshot);
+
+    const CGRect display = CGDisplayBounds(CGMainDisplayID());
+    if (!CGRectContainsPoint(display, location)) {
+      return std::nullopt;
+    }
+
+    return std::array<double, 2> {
+      (location.x - display.origin.x) / display.size.width,
+      (location.y - display.origin.y) / display.size.height
+    };
+  }
+
   bool is_screen_capture_allowed() {
     return screen_capture_allowed;
   }

src/platform/windows/misc.cpp

@@ -1877,4 +1877,37 @@ namespace platf {
   std::string resolve_render_device() {
     return {};
   }
+
+  std::optional<std::array<double, 2>> pointer_location() {
+    POINT cursor {};
+    if (!GetCursorPos(&cursor)) {
+      return std::nullopt;
+    }
+
+    // GetCursorPos answers in virtual-screen coordinates, which span every monitor, so a pointer
+    // on a second one lands outside the streamed display rather than nowhere. Only a pointer on
+    // the display being streamed means anything to the client.
+    MONITORINFO info {};
+    info.cbSize = sizeof(info);
+    const auto monitor = MonitorFromPoint(POINT {0, 0}, MONITOR_DEFAULTTOPRIMARY);
+    if (!monitor || !GetMonitorInfo(monitor, &info)) {
+      return std::nullopt;
+    }
+
+    const auto &bounds = info.rcMonitor;
+    const auto width = static_cast<double>(bounds.right - bounds.left);
+    const auto height = static_cast<double>(bounds.bottom - bounds.top);
+    if (width <= 0 || height <= 0) {
+      return std::nullopt;
+    }
+
+    if (cursor.x < bounds.left || cursor.x >= bounds.right || cursor.y < bounds.top || cursor.y >= bounds.bottom) {
+      return std::nullopt;
+    }
+
+    return std::array<double, 2> {
+      (cursor.x - bounds.left) / width,
+      (cursor.y - bounds.top) / height
+    };
+  }
 }  // namespace platf