Skip to content

Commit 21daf5c

Browse files
authored
feat: report clickable on android elements (#524)
* feat: report clickable on android elements * feat: report pane titles on android elements (#525)
1 parent 9f9966a commit 21daf5c

4 files changed

Lines changed: 115 additions & 7 deletions

File tree

‎agents/android/java/UiTreeSerializer.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,12 @@ private static JSONObject nodeToJson(AccessibilityNodeInfo node, int index) thro
154154
.put("x", bounds.left).put("y", bounds.top)
155155
.put("width", bounds.width()).put("height", bounds.height()));
156156

157+
// A pane title names a window-like region: Material3 sets "Dialog" on an AlertDialog,
158+
// "Bottom Sheet" on a ModalBottomSheet. API 28+.
159+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
160+
obj.put("pane-title", str(node.getPaneTitle()));
161+
}
162+
157163
// Sliders, progress bars and rating bars report their position here.
158164
AccessibilityNodeInfo.RangeInfo range = node.getRangeInfo();
159165
if (hasPosition(range)) {

‎devices/android.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1533,6 +1533,8 @@ type uiNode struct {
15331533
Range *types.ScreenElementRange `json:"range"`
15341534
// text can be typed into the node (EditText and the like)
15351535
Editable bool `json:"editable"`
1536+
// names a window-like region, e.g. "Dialog" on a Material3 AlertDialog
1537+
PaneTitle string `json:"pane-title"`
15361538
}
15371539

15381540
type uiHierarchy struct {
@@ -1582,17 +1584,18 @@ type androidNodeAttrs struct {
15821584
disabled bool
15831585
checked bool
15841586
selected bool
1587+
paneTitle string
15851588
rect types.ScreenElementRect
15861589
// value is a slider's position or a text field's content, "" when it has none
15871590
value string
15881591
sliderRange *types.ScreenElementRange
15891592
}
15901593

1591-
// isScreenElement reports whether the node is worth listing: it has text,
1594+
// isScreenElement reports whether the node is worth listing: it has text, a pane title,
15921595
// content-desc, hint, resource-id, or is interactable (clickable or checkable,
15931596
// e.g. Flutter icon-only buttons), and has a positive width and height.
15941597
func (a androidNodeAttrs) isScreenElement() bool {
1595-
isLabeled := a.text != "" || a.contentDesc != "" || a.hint != "" || a.resourceID != "" || a.value != ""
1598+
isLabeled := a.text != "" || a.contentDesc != "" || a.hint != "" || a.resourceID != "" || a.value != "" || a.paneTitle != ""
15961599
isInteractable := a.clickable || a.checkable
15971600
return (isLabeled || isInteractable) && a.rect.Width > 0 && a.rect.Height > 0
15981601
}
@@ -1614,6 +1617,9 @@ func (a androidNodeAttrs) toScreenElement(children []types.ScreenElement) types.
16141617
if a.resourceID != "" {
16151618
element.Identifier = &a.resourceID
16161619
}
1620+
if a.paneTitle != "" {
1621+
element.PaneTitle = &a.paneTitle
1622+
}
16171623
if a.value != "" {
16181624
element.Value = &a.value
16191625
}
@@ -1633,6 +1639,9 @@ func (a androidNodeAttrs) toScreenElement(children []types.ScreenElement) types.
16331639
if a.selected {
16341640
element.Selected = &isTrue
16351641
}
1642+
if a.clickable {
1643+
element.Clickable = &isTrue
1644+
}
16361645

16371646
// default type if class is empty
16381647
if element.Type == "" {
@@ -1714,6 +1723,7 @@ func collectUiNodeElements(nodes []uiNode) []types.ScreenElement {
17141723
disabled: !node.Enabled,
17151724
checked: node.Checked,
17161725
selected: node.Selected,
1726+
paneTitle: node.PaneTitle,
17171727
value: nodeValue(node),
17181728
sliderRange: node.Range,
17191729
rect: types.ScreenElementRect{

‎devices/android_elements_test.go‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -744,3 +744,93 @@ func TestUiNodeDecodesEditableFromAgentJson(t *testing.T) {
744744
t.Errorf("expected editable to be decoded")
745745
}
746746
}
747+
748+
// A Compose Button is a clickable android.view.View whose text lives on a child,
749+
// so clickable is the only hint that it is a button; it must surface Clickable=true.
750+
func TestCollectElementsMarksClickableNode(t *testing.T) {
751+
d := &AndroidDevice{}
752+
tree := uiAutomatorXmlNode{
753+
Class: "android.view.View",
754+
Clickable: "true",
755+
Bounds: "[0,0][140,140]",
756+
}
757+
758+
output := d.collectElements(tree)
759+
760+
if len(output) != 1 {
761+
t.Fatalf("expected 1 element, got %d: %+v", len(output), output)
762+
}
763+
if output[0].Clickable == nil || *output[0].Clickable != true {
764+
t.Errorf("expected Clickable to be true, got %+v", output[0].Clickable)
765+
}
766+
}
767+
768+
// A node that isn't clickable must leave Clickable unset (nil), matching omitempty.
769+
func TestCollectElementsLeavesNonClickableNodeUnset(t *testing.T) {
770+
d := &AndroidDevice{}
771+
tree := uiAutomatorXmlNode{
772+
Class: "android.widget.TextView",
773+
Text: "New Order",
774+
Bounds: "[0,0][140,140]",
775+
}
776+
777+
output := d.collectElements(tree)
778+
779+
if len(output) != 1 {
780+
t.Fatalf("expected 1 element, got %d: %+v", len(output), output)
781+
}
782+
if output[0].Clickable != nil {
783+
t.Errorf("expected Clickable to be nil for a non-clickable node, got %+v", *output[0].Clickable)
784+
}
785+
}
786+
787+
// A Material3 AlertDialog puts its pane title on a plain View that has no text of its own.
788+
// That node must be kept, with its pane title, so the dialog's content stays under it.
789+
func TestCollectUiNodeElementsKeepsPaneTitleNode(t *testing.T) {
790+
nodes := []uiNode{
791+
{
792+
Class: "android.view.View",
793+
PaneTitle: "Dialog",
794+
Rect: uiRect{X: 60, Y: 900, Width: 960, Height: 500},
795+
Children: []uiNode{
796+
{
797+
Class: "android.widget.TextView",
798+
Text: "You must be 18 or older to order a Dark Matter Martini",
799+
Rect: uiRect{X: 120, Y: 960, Width: 840, Height: 120},
800+
},
801+
},
802+
},
803+
}
804+
805+
output := collectUiNodeElements(nodes)
806+
807+
if len(output) != 1 {
808+
t.Fatalf("expected 1 element, got %d: %+v", len(output), output)
809+
}
810+
if output[0].PaneTitle == nil || *output[0].PaneTitle != "Dialog" {
811+
t.Errorf("expected PaneTitle to be Dialog, got %+v", output[0].PaneTitle)
812+
}
813+
if len(output[0].Children) != 1 {
814+
t.Errorf("expected the dialog text under the pane, got %+v", output[0].Children)
815+
}
816+
}
817+
818+
// A node without a pane title must leave PaneTitle unset (nil), matching omitempty.
819+
func TestCollectUiNodeElementsLeavesPaneTitleUnset(t *testing.T) {
820+
nodes := []uiNode{
821+
{
822+
Class: "android.widget.TextView",
823+
Text: "New Order",
824+
Rect: uiRect{X: 0, Y: 0, Width: 140, Height: 140},
825+
},
826+
}
827+
828+
output := collectUiNodeElements(nodes)
829+
830+
if len(output) != 1 {
831+
t.Fatalf("expected 1 element, got %d: %+v", len(output), output)
832+
}
833+
if output[0].PaneTitle != nil {
834+
t.Errorf("expected PaneTitle to be nil, got %+v", *output[0].PaneTitle)
835+
}
836+
}

‎types/screen.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,13 @@ type ScreenElement struct {
2929
Identifier *string `json:"identifier,omitempty"`
3030
Key *string `json:"key,omitempty"` // flutter debug builds: nearest enclosing widget Key('...')
3131
Rect ScreenElementRect `json:"rect"`
32-
Range *ScreenElementRange `json:"range,omitempty"` // android only: slider, progress bar, rating bar
33-
Focused *bool `json:"focused,omitempty"` // android tv, and ios when hasFocus is reported
34-
Enabled *bool `json:"enabled,omitempty"` // only set when false
35-
Checked *bool `json:"checked,omitempty"` // only set when true
36-
Selected *bool `json:"selected,omitempty"` // only set when true
32+
Range *ScreenElementRange `json:"range,omitempty"` // android only: slider, progress bar, rating bar
33+
Focused *bool `json:"focused,omitempty"` // android tv, and ios when hasFocus is reported
34+
Enabled *bool `json:"enabled,omitempty"` // only set when false
35+
Checked *bool `json:"checked,omitempty"` // only set when true
36+
Selected *bool `json:"selected,omitempty"` // only set when true
37+
Clickable *bool `json:"clickable,omitempty"` // android only: only set when true
38+
PaneTitle *string `json:"paneTitle,omitempty"` // android only: a window-like region's title, e.g. "Dialog"
3739
Children []ScreenElement `json:"children,omitempty"`
3840
}
3941

0 commit comments

Comments
 (0)