Fix macOS physical-input postconditions and add an advisory physical CI lane

.changeset/quiet-previews-scroll.md

@@ -0,0 +1,5 @@
+---
+'@fluentui-react-native/storybook-desktop-runtime': patch
+---
+
+Keep oversized macOS LiteUI stories within a scrollable viewport, matching Win32, so their first controls are not centered behind Storybook chrome and native keyboard focus can scroll later controls into view. Preserve Windows' existing non-scrolling story layout.

.changeset/tidy-physical-window.md

@@ -0,0 +1,7 @@
+---
+'@fluentui-react-native/storybook-desktop': patch
+---
+
+Establish and verify a centered macOS physical-lane Storybook window up to 1024x768, clamped within the largest visible screen frame with a one-point inset on each edge. Fail when no positive window fits or the host does not establish the exact requested rectangle. Leave accessible and compatibility lanes, native occlusion detection, and story assertions unchanged.
+
+Use the effective smoke input lane, including `DESKTOP_INPUT=physical` when no `--input` flag is supplied. Explicit input flags retain precedence over the environment.

.changeset/warm-hits-clear.md

@@ -0,0 +1,5 @@
+---
+'@fluentui-react-native/desktop-driver': patch
+---
+
+Keep hit-test windows from other macOS processes distinct while refusing same-process and unproven obstructions before activation, fall back to workspace activation when an acknowledged Accessibility request does not establish foreground state, wait for pointer placement without replaying motion, and use menu-independent document selection with verified focus, selection, and empty-value postconditions before reporting keyboard-clear success.

.github/workflows/pr.yml

@@ -231,6 +231,58 @@ jobs:
             apps/storybook/artifacts/desktop
             ${{ steps.desktop-driver.outputs.doctor-path }}
 
+  # An advisory pilot outside the aggregate PR check: hosted physical input is not yet qualified (#4441).
+  macos-storybook-physical:
+    name: macOS Storybook PR (physical)
+    runs-on: macos-26
+    timeout-minutes: 40
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up toolchain
+        uses: microsoft/react-native-test-app/.github/actions/setup-toolchain@1c99dd2947a77097bf937701e3dcc90aa109d17e # 5.4.15
+        with:
+          platform: macos
+          node-version: 22
+
+      - name: Install dependencies
+        run: yarn
+
+      - name: Build packages
+        run: yarn build
+
+      - name: Set up macOS Desktop Driver
+        id: desktop-driver
+        uses: ./.github/actions/setup-desktop-driver
+        with:
+          platform: macos
+
+      - name: Bundle macOS
+        run: yarn storybook bundle --macos
+        working-directory: apps/storybook
+
+      - name: Pod install
+        run: yarn storybook prep --macos
+        working-directory: apps/storybook
+
+      - name: Smoke test macOS Storybook physical input
+        run: yarn storybook smoke --macos --mode stories-and-tests --input physical
+        working-directory: apps/storybook
+        env:
+          CCACHE_DISABLE: 1
+
+      - name: Upload macOS Storybook physical artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: Storybook_macos_fabric_physical_${{ github.run_id }}_${{ github.run_attempt }}
+          path: |
+            apps/storybook/artifacts/macos
+            apps/storybook/artifacts/desktop
+            ${{ steps.desktop-driver.outputs.doctor-path }}
+          if-no-files-found: warn
+
   macos-paper-storybook:
     name: macOS Paper Storybook PR
     runs-on: macos-26

packages/agentic/desktop-driver/native/PROTOCOL.md

@@ -453,6 +453,15 @@ the element:
   the fallback. A check that refuses, such as `input-unavailable`, dispatches
   nothing and is `none`.
 
+The macOS helper requires a readable value before keyboard input and verifies
+that it becomes empty before returning success. It confirms keyboard focus
+and selects the document with Command-Up followed by Command-Shift-Down,
+without relying on the application's Select All menu command. When exposed,
+the selected text range must cover the complete value before Delete.
+Unacknowledged selection or retained text fails `input-failed`; already-posted
+keys retain their partial effect. These bounded reads do not replay input or
+permit a semantic fallback for `keyboard`.
+
 Any other `mechanism`, including one that is not a string, fails
 `invalid-params` with effect `none`. Node never sends `mechanism` unless the
 feature is negotiated, and treats a result without a recognized `mechanism`,

packages/agentic/desktop-driver/native/macos/README.md

@@ -21,6 +21,8 @@ Accessibility.
 | `Sources/DesktopDriverHost/main.swift` | Helper modes and top-level error handling                                   |
 | `Host.swift`                           | FDR1 hello, request queue, framing, cancellation, and disposal              |
 | `Driver.swift`                         | Command dispatch, capability reporting, and physical-input scopes           |
+| `ApplicationActivation.swift`          | Foreground postcondition and one workspace fallback after AX activation     |
+| `KeyboardClear.swift`                  | Keyboard focus, selection acknowledgment, and empty-value verification      |
 | `Application.swift`                    | Launch/attach leases, exact process/window identity, and ownership          |
 | `ApplicationTermination.swift`         | `terminateOwned`: quit, wait, kill, and verify the exit of an owned process |
 | `ApplicationIncarnation.swift`         | One-shot `--application-incarnation`: whether a recorded process exited     |
@@ -39,6 +41,7 @@ Accessibility.
 | `ObservationSelfTest.swift`            | Synthetic-tree, randomized differential, and command checks for `observe`   |
 | `ServiceIdentitySelfTest.swift`        | URL, claim, listener-policy, and live describe/verify checks                |
 | `InputIntegritySelfTest.swift`         | Recorded input: counted buttons and drags, cancelled clicks, fail-closed    |
+| `PhysicalInputSelfTest.swift`          | Hit-owner scope, foreground fallback, and verified keyboard clear           |
 | `HostLifecycleSelfTest.swift`          | In-process host: end of input, write failure, invalid JSON, queued work     |
 | `ApplicationTerminationSelfTest.swift` | Termination verification against fake and real child processes              |
 | `CaptureSelectionSelfTest.swift`       | Pure capture-window selection and `visible` without geometry                |
@@ -388,17 +391,37 @@ accessibility activation and reports physical success. A positive wheel
 `deltaX` scrolls right and a positive `deltaY` scrolls down, as on Windows. Quartz
 scrolls left and up for positive values, so both deltas are negated.
 
+After posting a pointer-move sequence, the helper allows about 200 ms of
+bounded position reads for the OS to acknowledge its destination. It never
+reposts the move to force acknowledgment. An unreadable or unsettled pointer
+fails `input-failed`; a click never posts its button press before the position
+is verified. Cancellation stops these reads and preserves the posted effect.
+
 The helper advertises `strictClear`. `clear` writes an empty `AXValue` when
 `AXUIElementIsAttributeSettable` reports the value settable, and otherwise
-activates the application, focuses the element, and presses Command-A and
-Delete under the physical input lock. A request may name `mechanism`:
+activates the application, focuses the element, selects the document with
+Command-Up then Command-Shift-Down, and presses Delete under the physical
+input lock. A request may name `mechanism`:
 `semantic` only writes `AXValue` and fails `element-not-interactable` with
 effect `none` when the value is not settable, and `keyboard` skips the write
 and needs the same physical input authority as the fallback. Every successful
 clear returns `{ "mechanism": "semantic" | "keyboard" }`. The `probe` feature
 is `true` even without Accessibility trust, because each mechanism reports its
 own refusal.
 
+Keyboard clear requires a readable value before posting input. It waits up to
+200 ms for native keyboard focus and posts the two document-navigation chords
+once, rather than relying on an application-provided Select All menu command.
+Paper's focused `NSTextField` can ignore Command-A even though the menu item
+is present; document navigation reaches the field editor without that menu
+route. When the provider exposes `AXSelectedTextRange`, clear waits up to
+200 ms for the complete UTF-16 value to be selected before posting Delete
+once. It then verifies an empty value for up to one second. An unreadable
+value is unsupported; missing focus fails before keys are posted.
+Unacknowledged selection or retained text fails `input-failed`, with the
+posted input's partial effect. Verification never replays keys or substitutes
+a semantic clear.
+
 A locked screen or a login session that is not the active console prevents
 applications from publishing complete accessibility windows and from receiving
 physical input. Launch, attach, window discovery, and new pointer, keyboard,
@@ -408,9 +431,11 @@ window-enumeration error. Physical input fails closed: a login session or pointe
 position the helper cannot read is a failure, not permission to proceed. Key and
 button releases remain allowed so recovery can complete while the session is locked.
 
-Activation sets the application's `AXFrontmost` attribute, which a trusted
-client applies immediately, and falls back to `NSRunningApplication.activate`
-only if that fails. Cooperative workspace activation was observed to wait about
+Activation first sets the application's `AXFrontmost` attribute and checks
+the foreground postcondition, not just the setter's acknowledgment. It falls
+back once to `NSRunningApplication.activate` when the write fails **or** the
+application is still not frontmost. The latter path is journaled as
+`accessibility+workspace`. Cooperative workspace activation was observed to wait about
 a second for the active application to yield. Activation completes when the
 application reports `AXFrontmost` and the primary window reports `AXMain`;
 a timeout names the application that kept keyboard focus. The window is also
@@ -423,6 +448,15 @@ A physical click first hit-tests its target. When another element owns the
 point, `element click intercepted` names the occluding element, or the external
 application and process that own it.
 
+Only a hit proven to belong to another process has an opaque window identity
+distinct from the target, allowing the agent to activate the target and recheck.
+A same-process covering window, or a hit whose owner cannot be established,
+retains the target's obstruction scope and is refused before activation, with
+effect `none`. Raising through a same-application popup could dismiss or reorder
+it, so that is not a physical click. A hit on the target's
+own window or chrome is not proof that its descendants receive input:
+genuinely covered or clipped controls remain refused.
+
 React Native macOS does not move keyboard focus on ordinary mouse-down. Portable
 tests should assert the platform's actual activation behavior rather than
 copying a Windows focus expectation.

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift

@@ -329,6 +329,21 @@ final class AccessibilityEngine {
     return axNumber(try copyOptionalAXAttribute(record.element, kAXFocusedAttribute))?.boolValue ?? false
   }
 
+  func selectedTextRange(_ elementID: String) throws -> CFRange? {
+    let record = try requireRecord(elementID)
+    try validateLive(record)
+    guard
+      let raw = try copyOptionalAXAttribute(record.element, kAXSelectedTextRangeAttribute),
+      CFGetTypeID(raw) == AXValueGetTypeID()
+    else {
+      return nil
+    }
+    let value = unsafeBitCast(raw, to: AXValue.self)
+    guard AXValueGetType(value) == .cfRange else { return nil }
+    var range = CFRange()
+    return AXValueGetValue(value, .cfRange, &range) ? range : nil
+  }
+
   func accessibilityClick(_ elementID: String, token: CancellationToken) throws {
     try requireAccessibility("Accessibility click")
     try token.throwIfCancelled()
@@ -995,23 +1010,29 @@ final class AccessibilityEngine {
     knownRecord(element)?.parentID
   }
 
-  private func externalWindowSnapshot(
+  func externalWindowSnapshot(
     window: WindowRecord,
     hit: AXUIElement
   ) -> ElementSnapshot {
+    let externalID = IdentifierGenerator.shared.next("external-window")
     let screenRect = (try? axFrame(hit)) ?? .zero
     let windowFrame = (try? window.currentFrame()) ?? .zero
     var ownerID: pid_t = 0
     var name = "External desktop window"
-    if AXUIElementGetPid(hit, &ownerID) == .success {
-      let owner = processIdentity(ownerID)
-      let ownerName = owner["name"] as? String ?? "unknown application"
-      name = ownerID == window.processID
-        ? "Another window of the target application"
-        : "External desktop window of \(ownerName) (pid \(ownerID))"
+    // Same-process and unproven owners retain the agent's no-activation obstruction scope.
+    var hitWindowID = window.id
+    if AXUIElementGetPid(hit, &ownerID) == .success && ownerID > 0 {
+      if ownerID == window.processID {
+        name = "Another window of the target application"
+      } else {
+        let owner = processIdentity(ownerID)
+        let ownerName = owner["name"] as? String ?? "unknown application"
+        name = "External desktop window of \(ownerName) (pid \(ownerID))"
+        hitWindowID = externalID
+      }
     }
     return ElementSnapshot(
-      id: IdentifierGenerator.shared.next("external-window"),
+      id: externalID,
       automationID: nil,
       checked: .unsupported("An external desktop window does not expose checked state."),
       enabled: .supported(true),
@@ -1032,7 +1053,7 @@ final class AccessibilityEngine {
       text: nil,
       value: nil,
       visible: .supported(true),
-      windowID: window.id
+      windowID: hitWindowID
     )
   }
 

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/ApplicationActivation.swift

@@ -0,0 +1,26 @@
+import ApplicationServices
+
+struct ApplicationActivation {
+  let method: String
+  let accessibilityError: AXError?
+}
+
+func activateApplication(
+  isFrontmost: () -> Bool,
+  accessibility: () -> AXError,
+  workspace: () -> Void
+) -> ApplicationActivation {
+  if isFrontmost() {
+    return ApplicationActivation(method: "already-frontmost", accessibilityError: nil)
+  }
+  let error = accessibility()
+  if error != .success {
+    workspace()
+    return ApplicationActivation(method: "workspace", accessibilityError: error)
+  }
+  if !isFrontmost() {
+    workspace()
+    return ApplicationActivation(method: "accessibility+workspace", accessibilityError: nil)
+  }
+  return ApplicationActivation(method: "accessibility", accessibilityError: nil)
+}

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift

@@ -381,17 +381,16 @@ final class Driver {
     func isFrontmost() -> Bool {
       axNumber(try? copyOptionalAXAttribute(applicationElement, kAXFrontmostAttribute))?.boolValue == true
     }
-    if !isFrontmost() {
-      // Trusted Accessibility clients activate immediately. NSRunningApplication activation is cooperative on current
-      // macOS releases and was observed to wait about a second for the active application to yield.
-      let frontmostError = AXUIElementSetAttributeValue(applicationElement, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
-      diagnostics["method"] = frontmostError == .success ? "accessibility" : "workspace"
-      if frontmostError != .success {
-        diagnostics["accessibilityError"] = axErrorName(frontmostError)
-        _ = application.activate(options: [])
-      }
-    } else {
-      diagnostics["method"] = "already-frontmost"
+    let activation = activateApplication(
+      isFrontmost: isFrontmost,
+      accessibility: {
+        AXUIElementSetAttributeValue(applicationElement, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
+      },
+      workspace: { _ = application.activate(options: []) }
+    )
+    diagnostics["method"] = activation.method
+    if let error = activation.accessibilityError {
+      diagnostics["accessibilityError"] = axErrorName(error)
     }
     _ = AXUIElementSetAttributeValue(window.element, kAXMainAttribute as CFString, kCFBooleanTrue)
     let raised = AXUIElementPerformAction(window.element, kAXRaiseAction as CFString)
@@ -582,9 +581,16 @@ final class Driver {
     }
     try withPhysicalInput(token: token) {
       try activate(window, token: token)
-      try accessibility.setFocus(elementID)
-      try token.wait(milliseconds: 16)
-      try input.pressCommandADelete(token: token)
+      try clearWithKeyboard(
+        token: token,
+        focus: { try accessibility.setFocus(elementID) },
+        hasFocus: { try accessibility.hasFocus(elementID) },
+        value: { try accessibility.snapshot(elementID: elementID, window: window, token: token).value },
+        selectedRange: { try accessibility.selectedTextRange(elementID) },
+        selectAll: { try input.selectAll(token: token) },
+        delete: { try input.deleteSelection(token: token) },
+        wait: { try token.wait(milliseconds: $0) }
+      )
     }
     return "keyboard"
   }

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Input.swift

@@ -120,12 +120,17 @@ final class InputController {
     }
     pointer = target
     try token.wait(milliseconds: 16)
-    guard let observed = environment.pointerLocation() else {
-      try fail(ErrorCode.inputFailed, "macOS did not report the pointer position, so the move to the requested native element coordinate is unverified.")
-    }
-    if abs(observed.x - target.x) > 2 || abs(observed.y - target.y) > 2 {
-      try fail(ErrorCode.inputFailed, "macOS did not move the pointer to the requested native element coordinate.")
+    for attempt in 0..<12 {
+      try token.throwIfCancelled()
+      guard let observed = environment.pointerLocation() else {
+        try fail(ErrorCode.inputFailed, "macOS did not report the pointer position, so the move to the requested native element coordinate is unverified.")
+      }
+      if abs(observed.x - target.x) <= 2 && abs(observed.y - target.y) <= 2 {
+        return
+      }
+      if attempt + 1 < 12 { try token.wait(milliseconds: 16) }
     }
+    try fail(ErrorCode.inputFailed, "macOS did not move the pointer to the requested native element coordinate.")
   }
 
   func pointerDown(_ button: Int, token: CancellationToken) throws {
@@ -230,8 +235,13 @@ final class InputController {
     }
   }
 
-  func pressCommandADelete(token: CancellationToken) throws {
-    try chord([.keyCode(55), .keyCode(0)], token: token)
+  func selectAll(token: CancellationToken) throws {
+    // Text-system navigation does not depend on the application's Select All menu command.
+    try chord([.keyCode(55), .keyCode(126)], token: token)
+    try chord([.keyCode(55), .keyCode(56), .keyCode(125)], token: token)
+  }
+
+  func deleteSelection(token: CancellationToken) throws {
     try chord([.keyCode(51)], token: token)
   }