Skip to content

Commit 1fd3152

Browse files
omnarayanomnarayan
committed
Mirror only the screen the app is actually on
iOS does not tear a screen down when the app navigates away from it, and XCUITest keeps reporting it. The mirror was faithful to that, so a login form — credentials still sitting in its fields — appeared on the product list and again in the cart, indistinguishable from what was in front of the user. An agent driven through the app found the dead Sign In button, clicked it, and got nothing; asked whether login had succeeded, it could not tell. A captured flow would record a selector resolving to a screen nobody can see, and carry that to real hardware. Nothing in a node separates the live screen from the dead one. hittable reads false for both, and both claim the full frame. Sibling order is the only signal — the view the app moved to is reported last — so among the full-screen siblings only the last survives. Windows are exempt, and that exemption is the point rather than a caveat. The keyboard lives inside the app's Window, which is itself full-screen and reported first, so culling on geometry alone would have taken a keyboard that is plainly on screen along with the dead views; that was measured before the rule was written, not after it broke. The exemption also makes the rule fail safe, since an app whose screens really are Windows keeps every node it had. Culled in the runner rather than in the browser: the console, the device page and Flow Capture all read this tree, and it is Flow Capture where a ghost selector survives longest before anyone notices.
1 parent a90d37b commit 1fd3152

4 files changed

Lines changed: 253 additions & 1 deletion

File tree

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,11 @@ Arm **Assert** and the next tap records an assertion instead of tapping. A point
8484
to nothing addressable is refused rather than recorded as a coordinate — "there are pixels here"
8585
is not worth putting in a flow.
8686
87+
**Only the screen you are on is mirrored.** iOS keeps a screen in the hierarchy after the app
88+
navigates away from it, so the tree reports views nobody can see — a login form, credentials
89+
still in its fields, present on every screen that follows. Those are culled, because a selector
90+
that resolves to an invisible screen fails silently here and again on real hardware.
91+
8792
**Text fields are real `<input>` elements**, so `fill()`, `inputValue()` and `toHaveValue()` work
8893
as they would on any page — and so does the `browser_type` an AI agent reaches for first. A
8994
field's contents live in its value, not its text, which is the one place the mirror departs from

‎internal/runner/occlusion.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package runner
2+
3+
// screenEpsilon absorbs the sub-point rounding in reported frames, so a
4+
// view one hundredth of a point short of the screen still counts as
5+
// covering it.
6+
const screenEpsilon = 1.0
7+
8+
// OnScreen drops the app views that a later view has replaced.
9+
//
10+
// iOS does not tear down a screen when the app navigates away from it:
11+
// the old view stays in the hierarchy and XCUITest keeps reporting it. A
12+
// mirror faithful to that tree therefore offers a login form — with the
13+
// credentials still in its fields — on every screen that follows,
14+
// indistinguishable from what is actually in front of the user. A driver
15+
// asked to sign in when already signed in finds the dead Sign In button,
16+
// clicks it, and nothing happens; a captured flow records a selector that
17+
// resolves to a screen nobody can see. Neither failure announces itself.
18+
//
19+
// Nothing in the node tells the two apart: hittable reads false for the
20+
// live screen and the dead one alike, and both claim the full screen.
21+
// Sibling order is the only signal — the view the app moved to is
22+
// reported last — so among the full-screen siblings only the last
23+
// survives.
24+
//
25+
// Windows are exempt, and that exemption is the whole subtlety. The
26+
// keyboard lives inside the app's Window, which is itself full-screen and
27+
// reported first; culling by geometry alone would take a keyboard that is
28+
// plainly on screen along with the dead views. Excluding Windows also
29+
// makes the rule fail safe: an app whose screens really are Windows keeps
30+
// every node it had before.
31+
func OnScreen(nodes []Node) []Node {
32+
if len(nodes) == 0 {
33+
return nodes
34+
}
35+
replaced := supersededViews(nodes)
36+
if len(replaced) == 0 {
37+
return nodes
38+
}
39+
doomed := withDescendants(nodes, replaced)
40+
kept := make([]Node, 0, len(nodes))
41+
for _, n := range nodes {
42+
if !doomed[n.Index] {
43+
kept = append(kept, n)
44+
}
45+
}
46+
return kept
47+
}
48+
49+
// supersededViews returns the indices of every full-screen view the app
50+
// has navigated away from — all but the last of them.
51+
func supersededViews(nodes []Node) map[int]bool {
52+
root := nodes[0]
53+
var fullScreen []int
54+
for _, n := range nodes[1:] {
55+
if n.Type == "Window" || n.ParentIndex == nil || *n.ParentIndex != root.Index {
56+
continue
57+
}
58+
if covers(n.Frame, root.Frame) {
59+
fullScreen = append(fullScreen, n.Index)
60+
}
61+
}
62+
if len(fullScreen) < 2 {
63+
return nil
64+
}
65+
out := make(map[int]bool, len(fullScreen)-1)
66+
for _, idx := range fullScreen[:len(fullScreen)-1] {
67+
out[idx] = true
68+
}
69+
return out
70+
}
71+
72+
// covers reports whether inner spans the whole of outer.
73+
func covers(inner, outer Rect) bool {
74+
return inner.X <= outer.X+screenEpsilon &&
75+
inner.Y <= outer.Y+screenEpsilon &&
76+
inner.X+inner.Width >= outer.X+outer.Width-screenEpsilon &&
77+
inner.Y+inner.Height >= outer.Y+outer.Height-screenEpsilon
78+
}
79+
80+
// withDescendants grows a set of node indices to include everything
81+
// beneath them. Children may appear before their parents in the slice, so
82+
// the walk repeats until it stops growing rather than assuming an order.
83+
func withDescendants(nodes []Node, seed map[int]bool) map[int]bool {
84+
out := make(map[int]bool, len(seed))
85+
for idx := range seed {
86+
out[idx] = true
87+
}
88+
for grew := true; grew; {
89+
grew = false
90+
for _, n := range nodes {
91+
if n.ParentIndex == nil || out[n.Index] || !out[*n.ParentIndex] {
92+
continue
93+
}
94+
out[n.Index] = true
95+
grew = true
96+
}
97+
}
98+
return out
99+
}

‎internal/runner/occlusion_test.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package runner
2+
3+
import "testing"
4+
5+
// fullFrame is the frame every full-screen view in these tests claims.
6+
var fullFrame = Rect{Width: 402, Height: 874}
7+
8+
func at(i int) *int { return &i }
9+
10+
// view builds a child of parent; a nil parent means the root.
11+
func view(i int, typ, label string, frame Rect, parent *int) Node {
12+
return Node{Index: i, Type: typ, Label: label, Frame: frame, ParentIndex: parent}
13+
}
14+
15+
func labelsOf(nodes []Node) []string {
16+
out := make([]string, 0, len(nodes))
17+
for _, n := range nodes {
18+
out = append(out, n.Label)
19+
}
20+
return out
21+
}
22+
23+
func sameLabels(a, b []string) bool {
24+
if len(a) != len(b) {
25+
return false
26+
}
27+
for i := range a {
28+
if a[i] != b[i] {
29+
return false
30+
}
31+
}
32+
return true
33+
}
34+
35+
func TestOnScreen(t *testing.T) {
36+
root := view(0, "Application", "app", fullFrame, nil)
37+
half := Rect{Y: 400, Width: 402, Height: 474}
38+
39+
tests := []struct {
40+
name string
41+
nodes []Node
42+
want []string
43+
}{
44+
{
45+
name: "empty tree is returned untouched",
46+
},
47+
{
48+
name: "a lone screen is never culled",
49+
nodes: []Node{root, view(1, "Other", "only", fullFrame, at(0))},
50+
want: []string{"app", "only"},
51+
},
52+
{
53+
// The case this exists for: the app navigated, and iOS kept
54+
// the fullFrame it left behind.
55+
name: "the screen navigated away from is dropped",
56+
nodes: []Node{
57+
root,
58+
view(1, "Other", "dead", fullFrame, at(0)),
59+
view(2, "Other", "live", fullFrame, at(0)),
60+
},
61+
want: []string{"app", "live"},
62+
},
63+
{
64+
name: "a dropped screen takes its whole subtree",
65+
nodes: []Node{
66+
root,
67+
view(1, "Other", "dead", fullFrame, at(0)),
68+
view(2, "Button", "ghost", half, at(1)),
69+
view(3, "Other", "deeper", half, at(2)),
70+
view(4, "Other", "live", fullFrame, at(0)),
71+
},
72+
want: []string{"app", "live"},
73+
},
74+
{
75+
// Children can be reported before their parents, so the walk
76+
// must not assume slice order.
77+
name: "descendants listed before their parent are still dropped",
78+
nodes: []Node{
79+
root,
80+
view(3, "Other", "deeper", half, at(2)),
81+
view(2, "Button", "ghost", half, at(1)),
82+
view(1, "Other", "dead", fullFrame, at(0)),
83+
view(4, "Other", "live", fullFrame, at(0)),
84+
},
85+
want: []string{"app", "live"},
86+
},
87+
{
88+
// The keyboard lives in the Window. Culling it would take a
89+
// keyboard that is plainly on screen.
90+
name: "a Window is never culled, whatever follows it",
91+
nodes: []Node{
92+
root,
93+
view(1, "Window", "window", fullFrame, at(0)),
94+
view(2, "Other", "dead", fullFrame, at(0)),
95+
view(3, "Other", "live", fullFrame, at(0)),
96+
},
97+
want: []string{"app", "window", "live"},
98+
},
99+
{
100+
name: "a partial overlay leaves the screen beneath it alone",
101+
nodes: []Node{
102+
root,
103+
view(1, "Other", "screen", fullFrame, at(0)),
104+
view(2, "Other", "sheet", half, at(0)),
105+
},
106+
want: []string{"app", "screen", "sheet"},
107+
},
108+
{
109+
name: "sub-point rounding still counts as full screen",
110+
nodes: []Node{
111+
root,
112+
view(1, "Other", "dead", Rect{X: 0.4, Width: 401.7, Height: 873.8}, at(0)),
113+
view(2, "Other", "live", fullFrame, at(0)),
114+
},
115+
want: []string{"app", "live"},
116+
},
117+
{
118+
// Only the root's own children are screens; a full-fullFrame view
119+
// nested deeper is content.
120+
name: "a full-screen node deeper in the tree is not a screen",
121+
nodes: []Node{
122+
root,
123+
view(1, "Other", "screen", fullFrame, at(0)),
124+
view(2, "Other", "inner", fullFrame, at(1)),
125+
},
126+
want: []string{"app", "screen", "inner"},
127+
},
128+
}
129+
130+
for _, tt := range tests {
131+
t.Run(tt.name, func(t *testing.T) {
132+
got := labelsOf(OnScreen(tt.nodes))
133+
if tt.want == nil {
134+
if len(got) != 0 {
135+
t.Fatalf("expected nothing, got %v", got)
136+
}
137+
return
138+
}
139+
if !sameLabels(got, tt.want) {
140+
t.Errorf("OnScreen() = %v, want %v", got, tt.want)
141+
}
142+
})
143+
}
144+
}

‎internal/runner/tree.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,11 @@ func (t *TreeClient) Snapshot(ctx context.Context, appBundleID string) ([]Node,
6161
if data == nil {
6262
return nil, fmt.Errorf("runner snapshot: empty response")
6363
}
64-
return convertNodes(data.Nodes), nil
64+
// Culled here rather than in the browser: the console, the device
65+
// page and Flow Capture all read this tree, and a flow recorded
66+
// against a screen nobody can see is the failure that survives to
67+
// real hardware.
68+
return OnScreen(convertNodes(data.Nodes)), nil
6569
}
6670

6771
func convertNodes(in []dlios.SnapshotNode) []Node {

0 commit comments

Comments
 (0)