Skip to content

Commit 7da8d77

Browse files
committed
Show an agent when the mirrored app is not in the foreground
A backgrounded iOS app keeps its whole accessibility tree, so the mirror still draws every control and an agent would still click one — onto the springboard, not the app. No signal in the node tree catches this: per-node hittable oscillates while a screen animates into the background and settles back to its foreground values, so a tree-only check is racy both ways. Read the app's own XCUIApplication.state, which the runner now reports on each snapshot (maestro-runner 5d782aa). It threads through as runner.Snapshot{Nodes, AppState} on the settle seam and the server's TreeSource, and the tree payload gains "foreground" and "appState". The page annotates the root name — "device screen … — app not in foreground" — so it rides into an agent's snapshot, and sets data-dd-foreground for tests. Annotated, never blocked: the tap still forwards, because a wrong state read must not strand a caller. Empty state (Android, or a runner that did not send one) reads as foreground. go.mod moves to the tagged maestro-runner pseudo-version that carries appState (no local replace). Coverage held: the testable seam is 100% (treePayload foreground mapping, SnapshotState delegators); the engine SnapshotState methods keep the e2e waiver. Verified: full suite green, mirror-fixtures 13/13 including the backgrounded-app annotation. Measured limit, worth knowing: a scoped XCUITest snapshot tends to foreground the target app, so under continuous polling a backgrounded app is pulled forward within a poll and the annotation is brief — exact at snapshot time and right for single-shot consumers, not a fight to keep an app backgrounded.
1 parent aed28ac commit 7da8d77

11 files changed

Lines changed: 151 additions & 44 deletions

File tree

‎examples/playwright/tests/platform/mirror-fixtures.spec.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,3 +198,16 @@ test('the canvas takes the device dimensions from the tree, with no video', asyn
198198
expect(size.w).toBe(Math.round(app.width));
199199
expect(size.h).toBe(Math.round(app.height));
200200
});
201+
202+
test('a backgrounded app is announced in the root name, not hidden', async ({ page }) => {
203+
const tree = await serveMirror(page, 'products');
204+
// The same screen, now reported backgrounded. iOS keeps a backgrounded
205+
// app's whole tree, so every control is still here — the mirror must
206+
// annotate, not hide, so an agent knows the screen is not on the device.
207+
await page.route('**/api/devices/*/tree*', (r) =>
208+
r.fulfill({ json: { ...tree, foreground: false, appState: 'runningBackground' } }));
209+
await expect(page.locator('#mirror')).toHaveAttribute('aria-label', /app not in foreground/, { timeout: 5_000 });
210+
await expect(page.locator('#mirror')).toHaveAttribute('data-dd-foreground', 'false');
211+
// Annotated, never hidden: the controls are all still present and clickable.
212+
await expect(page.locator('#mirror [data-dd-node]').first()).toBeVisible();
213+
});

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.26
44

55
require (
66
github.com/coder/websocket v1.8.15
7-
github.com/devicelab-dev/maestro-runner v1.1.25-0.20260818192022-2ad25f651253
7+
github.com/devicelab-dev/maestro-runner v1.1.25-0.20260821233021-5d782aa65f33
88
google.golang.org/grpc v1.83.0
99
google.golang.org/protobuf v1.36.12
1010
gopkg.in/yaml.v3 v3.0.1

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
33
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
44
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
55
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
6-
github.com/devicelab-dev/maestro-runner v1.1.25-0.20260818192022-2ad25f651253 h1:X8NtuvyUj/HNTGn6bAeFEaXaMaRTwI3aAe4/uDwgY5Q=
7-
github.com/devicelab-dev/maestro-runner v1.1.25-0.20260818192022-2ad25f651253/go.mod h1:DMSQv0AMy2xs1COCdt/eIbUUmGzyLX35KmgKx/uJ0Xs=
6+
github.com/devicelab-dev/maestro-runner v1.1.25-0.20260821233021-5d782aa65f33 h1:sbVSlYcCp+LXd1OpeF93DjCA5VU58re1/0Q1wh2tnGY=
7+
github.com/devicelab-dev/maestro-runner v1.1.25-0.20260821233021-5d782aa65f33/go.mod h1:DMSQv0AMy2xs1COCdt/eIbUUmGzyLX35KmgKx/uJ0Xs=
88
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
99
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
1010
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=

‎internal/runner/androidtree.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,16 @@ func (e *AndroidEngine) Snapshot(_ context.Context, _ string) ([]Node, error) {
155155
return convertAndroidElements(elems, w, h), nil
156156
}
157157

158+
// SnapshotState satisfies engineAPI. Like the rest of AndroidEngine it runs
159+
// only against a real emulator (coverage waiver, as AndroidEngine.Snapshot).
160+
// Android's AccessibilityService page source carries no app-lifecycle state,
161+
// so AppState is empty and a caller treats "unknown" and empty alike — the
162+
// foreground annotation is simply absent, never a false claim.
163+
func (e *AndroidEngine) SnapshotState(ctx context.Context, appBundleID string) (Snapshot, error) {
164+
nodes, err := e.Snapshot(ctx, appBundleID)
165+
return Snapshot{Nodes: nodes}, err
166+
}
167+
158168
// Stop tears the session down, gracefully first.
159169
func (e *AndroidEngine) Stop(context.Context) error {
160170
e.mu.Lock()

‎internal/runner/settle.go‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,22 @@ import (
66
"time"
77
)
88

9+
// Snapshot is a UI tree together with the app-lifecycle state that came
10+
// with it — the target's XCUIApplication.state on iOS, empty where the
11+
// platform does not report one. The state travels with the tree because
12+
// it is the same dump: no signal in the nodes reports foreground/
13+
// background reliably (per-node hittable oscillates while a screen
14+
// animates into the background), so a caller reads AppState, not the
15+
// tree, to know the app is frontmost.
16+
type Snapshot struct {
17+
Nodes []Node
18+
AppState string
19+
}
20+
921
// Snapshotter is the one tree operation settling needs. Narrow on purpose:
1022
// it lets the settle logic be exercised against a scripted sequence of
1123
// trees, with no engine and no device.
12-
type Snapshotter func(ctx context.Context) ([]Node, error)
24+
type Snapshotter func(ctx context.Context) (Snapshot, error)
1325

1426
// SettleOptions bounds a settle wait. Zero values take the defaults below.
1527
type SettleOptions struct {
@@ -64,19 +76,19 @@ func (o SettleOptions) withDefaults() SettleOptions {
6476
//
6577
// An empty `after` asks only for quiet, which is what a caller wants on
6678
// first contact, before it has any hash to compare against.
67-
func Settle(ctx context.Context, snap Snapshotter, after string, opts SettleOptions) ([]Node, error) {
79+
func Settle(ctx context.Context, snap Snapshotter, after string, opts SettleOptions) (Snapshot, error) {
6880
opts = opts.withDefaults()
6981
deadline := time.Now().Add(opts.Cap)
70-
var last []Node
82+
var last Snapshot
7183
lastScreen, quiet := "", 0
7284
for {
73-
nodes, err := snap(ctx)
85+
got, err := snap(ctx)
7486
if err != nil {
75-
return nil, err
87+
return Snapshot{}, err
7688
}
77-
last = nodes
78-
quiet, lastScreen = countQuiet(nodes, lastScreen, quiet)
79-
changed := after == "" || InteractionHash(nodes) != after
89+
last = got
90+
quiet, lastScreen = countQuiet(got.Nodes, lastScreen, quiet)
91+
changed := after == "" || InteractionHash(got.Nodes) != after
8092
if changed && quiet >= opts.Quiet {
8193
return last, nil
8294
}
@@ -198,8 +210,8 @@ func awaitAppeared(ctx context.Context, snap Snapshotter, opts LaunchOptions) er
198210
deadline := time.Now().Add(opts.Appear)
199211
var lastErr error
200212
for {
201-
nodes, err := snap(ctx)
202-
if err == nil && hasControl(nodes) {
213+
got, err := snap(ctx)
214+
if err == nil && hasControl(got.Nodes) {
203215
return nil
204216
}
205217
lastErr = err

‎internal/runner/settle_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,13 @@ func focused(label string) []Node {
2626
// took.
2727
func scripted(trees ...[]Node) (Snapshotter, *int) {
2828
calls := 0
29-
return func(ctx context.Context) ([]Node, error) {
29+
return func(ctx context.Context) (Snapshot, error) {
3030
i := calls
3131
calls++
3232
if i >= len(trees) {
3333
i = len(trees) - 1
3434
}
35-
return trees[i], nil
35+
return Snapshot{Nodes: trees[i]}, nil
3636
}, &calls
3737
}
3838

@@ -91,8 +91,8 @@ func TestSettle(t *testing.T) {
9191
if err != nil {
9292
t.Fatalf("unexpected error: %v", err)
9393
}
94-
if got[0].Label != tc.wantLabel {
95-
t.Errorf("settled on %q, want %q", got[0].Label, tc.wantLabel)
94+
if got.Nodes[0].Label != tc.wantLabel {
95+
t.Errorf("settled on %q, want %q", got.Nodes[0].Label, tc.wantLabel)
9696
}
9797
if *calls < tc.minCalls {
9898
t.Errorf("took %d samples, want at least %d", *calls, tc.minCalls)
@@ -103,7 +103,7 @@ func TestSettle(t *testing.T) {
103103

104104
func TestSettleSnapshotErrorIsReturned(t *testing.T) {
105105
boom := errors.New("engine down")
106-
snap := func(ctx context.Context) ([]Node, error) { return nil, boom }
106+
snap := func(ctx context.Context) (Snapshot, error) { return Snapshot{}, boom }
107107
if _, err := Settle(context.Background(), snap, "", fast); !errors.Is(err, boom) {
108108
t.Fatalf("got %v, want %v", err, boom)
109109
}
@@ -115,7 +115,7 @@ func TestSettleStopsOnCancel(t *testing.T) {
115115
cancel()
116116
// Cancelled before the first sleep: one sample, then out.
117117
got, err := Settle(ctx, snap, InteractionHash(screen("zzz")), SettleOptions{Interval: time.Hour, Cap: 2 * time.Hour})
118-
if err != nil || got == nil {
118+
if err != nil || got.Nodes == nil {
119119
t.Fatalf("cancel should still hand back the last tree, got %v, %v", got, err)
120120
}
121121
if *calls != 1 {
@@ -168,7 +168,7 @@ func TestAwaitLaunched(t *testing.T) {
168168
})
169169
t.Run("a snapshot error explains the timeout", func(t *testing.T) {
170170
boom := errors.New("engine down")
171-
snap := func(ctx context.Context) ([]Node, error) { return nil, boom }
171+
snap := func(ctx context.Context) (Snapshot, error) { return Snapshot{}, boom }
172172
if err := AwaitLaunched(context.Background(), snap, fastLaunch); !errors.Is(err, boom) {
173173
t.Fatalf("got %v, want %v", err, boom)
174174
}

‎internal/runner/tree.go‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,21 +51,30 @@ func NewTreeClient(host string, port int) *TreeClient {
5151
// Snapshot returns the full UI tree for appBundleID (empty = foreground app,
5252
// runner-side default).
5353
func (t *TreeClient) Snapshot(ctx context.Context, appBundleID string) ([]Node, error) {
54+
snap, err := t.SnapshotState(ctx, appBundleID)
55+
return snap.Nodes, err
56+
}
57+
58+
// SnapshotState is Snapshot plus the app's lifecycle state, read from the
59+
// same dump. A snapshot is not an interaction command, so the runner does
60+
// not activate the app first: AppState is its true frontmost/background
61+
// state, which the node tree cannot report reliably.
62+
func (t *TreeClient) SnapshotState(ctx context.Context, appBundleID string) (Snapshot, error) {
5463
data, err := t.client.Call(ctx, dlios.Command{
5564
Command: dlios.CmdSnapshot,
5665
AppBundleID: appBundleID,
5766
})
5867
if err != nil {
59-
return nil, fmt.Errorf("runner snapshot: %w", err)
68+
return Snapshot{}, fmt.Errorf("runner snapshot: %w", err)
6069
}
6170
if data == nil {
62-
return nil, fmt.Errorf("runner snapshot: empty response")
71+
return Snapshot{}, fmt.Errorf("runner snapshot: empty response")
6372
}
6473
// Culled here rather than in the browser: the console, the device
6574
// page and Flow Capture all read this tree, and a flow recorded
6675
// against a screen nobody can see is the failure that survives to
6776
// real hardware.
68-
return OnScreen(convertNodes(data.Nodes)), nil
77+
return Snapshot{Nodes: OnScreen(convertNodes(data.Nodes)), AppState: data.AppState}, nil
6978
}
7079

7180
func convertNodes(in []dlios.SnapshotNode) []Node {
@@ -102,9 +111,10 @@ type Engine struct {
102111

103112
// StartEngine builds (first run only) and launches the runner on udid.
104113
//
105-
// Coverage waiver: StartEngine, Engine.Snapshot, and Engine.Stop wrap the
106-
// xcodebuild-driven runner lifecycle and only execute against a real booted
107-
// simulator; they are exercised by end-to-end runs, not unit tests.
114+
// Coverage waiver: StartEngine, Engine.Snapshot, Engine.SnapshotState, and
115+
// Engine.Stop wrap the xcodebuild-driven runner lifecycle and only execute
116+
// against a real booted simulator; they are exercised by end-to-end runs,
117+
// not unit tests.
108118
func StartEngine(ctx context.Context, udid string) (*Engine, error) {
109119
// Embedded, not installed: the runner source ships inside the pinned
110120
// maestro-runner module, so the runner DeviceDeck builds always
@@ -129,6 +139,11 @@ func (e *Engine) Snapshot(ctx context.Context, appBundleID string) ([]Node, erro
129139
return e.tree.Snapshot(ctx, appBundleID)
130140
}
131141

142+
// SnapshotState fetches the UI tree and the app's lifecycle state.
143+
func (e *Engine) SnapshotState(ctx context.Context, appBundleID string) (Snapshot, error) {
144+
return e.tree.SnapshotState(ctx, appBundleID)
145+
}
146+
132147
// Stop shuts the runner down, gracefully first.
133148
func (e *Engine) Stop(ctx context.Context) error {
134149
return dlios.GracefulShutdown(ctx, e.client, e.handle)
@@ -138,6 +153,7 @@ func (e *Engine) Stop(ctx context.Context) error {
138153
// by test fakes so cache behavior is testable without xcodebuild.
139154
type engineAPI interface {
140155
Snapshot(ctx context.Context, appBundleID string) ([]Node, error)
156+
SnapshotState(ctx context.Context, appBundleID string) (Snapshot, error)
141157
Stop(ctx context.Context) error
142158
}
143159

@@ -172,21 +188,28 @@ func NewEngines() *Engines {
172188
// alive; restarting a live engine over an app-level error (APP_NOT_RUNNING,
173189
// a caught in-runner exception) just burns the startup cost for nothing.
174190
func (s *Engines) Snapshot(ctx context.Context, udid, appBundleID string) ([]Node, error) {
191+
snap, err := s.SnapshotState(ctx, udid, appBundleID)
192+
return snap.Nodes, err
193+
}
194+
195+
// SnapshotState is Snapshot carrying the app's lifecycle state, with the
196+
// same evict-and-retry behavior.
197+
func (s *Engines) SnapshotState(ctx context.Context, udid, appBundleID string) (Snapshot, error) {
175198
e, err := s.engine(ctx, udid)
176199
if err != nil {
177-
return nil, err
200+
return Snapshot{}, err
178201
}
179-
nodes, err := e.Snapshot(ctx, appBundleID)
202+
snap, err := e.SnapshotState(ctx, appBundleID)
180203
var runnerErr *dlios.RunnerError
181204
if err == nil || ctx.Err() != nil || errors.As(err, &runnerErr) {
182-
return nodes, err
205+
return snap, err
183206
}
184207
slog.Warn("tree snapshot failed, restarting engine", "udid", udid, "error", err)
185208
s.evict(ctx, udid, e)
186209
if e, err = s.engine(ctx, udid); err != nil {
187-
return nil, fmt.Errorf("restart tree engine: %w", err)
210+
return Snapshot{}, fmt.Errorf("restart tree engine: %w", err)
188211
}
189-
return e.Snapshot(ctx, appBundleID)
212+
return e.SnapshotState(ctx, appBundleID)
190213
}
191214

192215
// evict drops failed from the cache and stops it — unless a concurrent

‎internal/runner/tree_test.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,10 @@ type fakeEngine struct {
107107
}
108108

109109
func (f *fakeEngine) Snapshot(context.Context, string) ([]Node, error) { return f.nodes, f.err }
110-
func (f *fakeEngine) Stop(context.Context) error { f.stopped = true; return nil }
110+
func (f *fakeEngine) SnapshotState(context.Context, string) (Snapshot, error) {
111+
return Snapshot{Nodes: f.nodes}, f.err
112+
}
113+
func (f *fakeEngine) Stop(context.Context) error { f.stopped = true; return nil }
111114

112115
func TestEnginesCachesPerUDID(t *testing.T) {
113116
started := map[string]int{}

‎internal/server/server.go‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ type FrameSender interface {
5353
// TreeSource fetches the UI tree for a device.
5454
type TreeSource interface {
5555
Snapshot(ctx context.Context, udid, appBundleID string) ([]runner.Node, error)
56+
SnapshotState(ctx context.Context, udid, appBundleID string) (runner.Snapshot, error)
5657
}
5758

5859
// CaptureService records manual sessions as flows.
@@ -210,8 +211,8 @@ func (s *Server) handleLaunchApp(w http.ResponseWriter, r *http.Request) {
210211
// snapshotter binds the tree source to one device and app, which is the
211212
// shape the settle logic wants.
212213
func (s *Server) snapshotter(udid, app string) runner.Snapshotter {
213-
return func(ctx context.Context) ([]runner.Node, error) {
214-
return s.trees.Snapshot(ctx, udid, app)
214+
return func(ctx context.Context) (runner.Snapshot, error) {
215+
return s.trees.SnapshotState(ctx, udid, app)
215216
}
216217
}
217218

@@ -228,7 +229,7 @@ func (s *Server) handleScreenshot(w http.ResponseWriter, r *http.Request) {
228229
func (s *Server) handleTree(w http.ResponseWriter, r *http.Request) {
229230
udid := r.PathValue("udid")
230231
snap := s.snapshotter(udid, r.URL.Query().Get("app"))
231-
var nodes []runner.Node
232+
var got runner.Snapshot
232233
var err error
233234
// ?after=<interaction hash> turns the poll into a barrier: the
234235
// response is held until the screen has moved on from that hash and
@@ -237,9 +238,9 @@ func (s *Server) handleTree(w http.ResponseWriter, r *http.Request) {
237238
// that waits on the page's own requests — playwright-mcp does —
238239
// waits for the device without knowing it.
239240
if after, held := r.URL.Query()["after"]; held {
240-
nodes, err = runner.Settle(r.Context(), snap, after[0], s.settle)
241+
got, err = runner.Settle(r.Context(), snap, after[0], s.settle)
241242
} else {
242-
nodes, err = snap(r.Context())
243+
got, err = snap(r.Context())
243244
}
244245
if err != nil {
245246
// Logged, not just returned: a failed snapshot reaches the page
@@ -251,7 +252,7 @@ func (s *Server) handleTree(w http.ResponseWriter, r *http.Request) {
251252
httpError(w, http.StatusBadGateway, err)
252253
return
253254
}
254-
writeJSON(w, treePayload(nodes))
255+
writeJSON(w, treePayload(got))
255256
}
256257

257258
// treePayload is the tree as every consumer receives it. The hashes
@@ -260,26 +261,33 @@ func (s *Server) handleTree(w http.ResponseWriter, r *http.Request) {
260261
// "hash" is the screen, for settledness and the fingerprint;
261262
// "interaction" adds focus, and is what a client hands back as ?after=
262263
// so that moving between fields counts as a change.
263-
func treePayload(nodes []runner.Node) map[string]any {
264+
func treePayload(snap runner.Snapshot) map[string]any {
265+
nodes := snap.Nodes
264266
if nodes == nil {
265267
nodes = []runner.Node{}
266268
}
267269
return map[string]any{
268270
"nodes": nodes,
269271
"hash": runner.ScreenHash(nodes),
270272
"interaction": runner.InteractionHash(nodes),
273+
// foreground is false only when the app reports a background or
274+
// not-running state. An empty state — Android, or a runner that
275+
// did not send one — is treated as foreground: absence of the
276+
// signal must not read as "backgrounded" and block nothing.
277+
"foreground": snap.AppState == "" || snap.AppState == "runningForeground",
278+
"appState": snap.AppState,
271279
}
272280
}
273281

274282
// FirstTree is the settled tree for a device page, ready to inline. It
275283
// waits for quiet alone — there is no earlier screen to have moved on
276284
// from — under the same cap as any other held tree.
277285
func (s *Server) FirstTree(ctx context.Context, udid, app string) ([]byte, error) {
278-
nodes, err := runner.Settle(ctx, s.snapshotter(udid, app), "", s.settle)
286+
got, err := runner.Settle(ctx, s.snapshotter(udid, app), "", s.settle)
279287
if err != nil {
280288
return nil, err
281289
}
282-
return json.Marshal(treePayload(nodes))
290+
return json.Marshal(treePayload(got))
283291
}
284292

285293
type tapRequest struct {

0 commit comments

Comments
 (0)