{"id":"https://openalex.org/W7166715213","doi":"https://doi.org/10.1145/3808134","title":"Look Before You Leap: Context-Sensitive GUI Grounding for Boosting Automated Extended Reality (XR) Testing","display_name":"Look Before You Leap: Context-Sensitive GUI Grounding for Boosting Automated Extended Reality (XR) Testing","publication_year":2026,"publication_date":"2026-06-30","ids":{"openalex":"https://openalex.org/W7166715213","doi":"https://doi.org/10.1145/3808134"},"language":"en","primary_location":{"id":"doi:10.1145/3808134","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3808134","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3808134","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100636489","display_name":"Shuqing Li","orcid":"https://orcid.org/0000-0001-6323-1402"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shuqing Li","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-6323-1402","affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111035261","display_name":"Binchang Li","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Binchang Li","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China"],"raw_orcid":"https://orcid.org/0009-0008-5995-4040","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084868951","display_name":"Yepang Liu","orcid":"https://orcid.org/0000-0001-8147-8126"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yepang Liu","raw_affiliation_strings":["Southern University of Science and Technology, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0001-8147-8126","affiliations":[{"raw_affiliation_string":"Southern University of Science and Technology, Shenzhen, China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061699251","display_name":"Cuiyun Gao","orcid":"https://orcid.org/0000-0003-4774-2434"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cuiyun Gao","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-4774-2434","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104386597","display_name":"Jianping Zhang","orcid":"https://orcid.org/0000-0002-8262-9608"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jianping Zhang","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0009-0005-5978-8888","affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034057959","display_name":"Shing-Chi Cheung","orcid":"https://orcid.org/0000-0002-3508-7172"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shing-Chi Cheung","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-3508-7172","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069596903","display_name":"Michael R. Lyu","orcid":"https://orcid.org/0000-0002-3666-5798"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Michael R. Lyu","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-3666-5798","affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.85865378,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"3","issue":"FSE","first_page":"2858","last_page":"2881"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10888","display_name":"Augmented Reality Applications","score":0.2718000113964081,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10888","display_name":"Augmented Reality Applications","score":0.2718000113964081,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.22190000116825104,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.11050000041723251,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/graphical-user-interface","display_name":"Graphical user interface","score":0.5342000126838684},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4586000144481659},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.3531000018119812},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.3294000029563904},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.3280999958515167},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.3278000056743622},{"id":"https://openalex.org/keywords/virtual-reality","display_name":"Virtual reality","score":0.3255000114440918},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.32350000739097595}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7452999949455261},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6366000175476074},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.5342000126838684},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4586000144481659},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3531000018119812},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.3294000029563904},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.3280999958515167},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.3278000056743622},{"id":"https://openalex.org/C194969405","wikidata":"https://www.wikidata.org/wiki/Q170519","display_name":"Virtual reality","level":2,"score":0.3255000114440918},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.32350000739097595},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30889999866485596},{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.2969000041484833},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C25344961","wikidata":"https://www.wikidata.org/wiki/Q192726","display_name":"Virtual machine","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.271699994802475},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C2778701210","wikidata":"https://www.wikidata.org/wiki/Q28130034","display_name":"Constructive","level":3,"score":0.26100000739097595}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3808134","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3808134","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3808134","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3808134","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W2141125339","https://openalex.org/W2158447593","https://openalex.org/W2241093273","https://openalex.org/W2345006418","https://openalex.org/W2794908093","https://openalex.org/W2959108026","https://openalex.org/W2963295463","https://openalex.org/W2993798883","https://openalex.org/W3043122026","https://openalex.org/W3048516901","https://openalex.org/W3098062821","https://openalex.org/W3194624166","https://openalex.org/W4238641215","https://openalex.org/W4283079003","https://openalex.org/W4287901611","https://openalex.org/W4296131098","https://openalex.org/W4312690238","https://openalex.org/W4313547589","https://openalex.org/W4313563530","https://openalex.org/W4384154362","https://openalex.org/W4385848842","https://openalex.org/W4386846781","https://openalex.org/W4388483659","https://openalex.org/W4389158376","https://openalex.org/W4399695837","https://openalex.org/W4402716147","https://openalex.org/W4402753581","https://openalex.org/W4412887717","https://openalex.org/W4417515183","https://openalex.org/W6948369276","https://openalex.org/W7077911270"],"related_works":[],"abstract_inverted_index":{"In":[0,112,248],"recent":[1,147],"years,":[2],"Extended":[3,263],"Reality":[4,264],"(XR)":[5],"has":[6],"emerged":[7],"as":[8,69,359,361],"a":[9,114,162,169,176,202,282],"transformative":[10],"technology,":[11],"offering":[12],"users":[13],"immersive":[14],"and":[15,60,103,108,124,135,159,186,211,219,226,299,347,364,380,433,446,452],"interactive":[16],"experiences":[17],"across":[18,404],"diversified":[19],"virtual":[20],"or":[21,428],"virtual-real":[22],"environments.":[23],"Users":[24],"can":[25,76,89,128,190],"interact":[26,90],"with":[27,175,400,493],"XR":[28,51,82,86,198,246,287],"applications":[29],"(apps)":[30],"through":[31],"interactable":[32,257,402,471],"GUI":[33,73,172,181,258,338,350,422],"elements":[34,351],"(IGEs)":[35],"on":[36,132,168,240,271,353,398,410],"the":[37,47,70,93,121,220,241,253,314,336,386,411,420,450,455,463,470,475,480],"stereoscopic":[38],"three-dimensional":[39],"(3D)":[40],"graphical":[41],"user":[42],"interface":[43],"(GUI).":[44],"IGE":[45,148,195,213,231,242,342,357],"constitutes":[46],"fundamental":[48],"element":[49,182,259,423],"of":[50,63,72,126,179,204,216,222,454,465,479],"GUI,":[52],"embodying":[53],"rich":[54],"semantic":[55,293,371],"information.":[56],"The":[57,145],"accurate":[58,122,230],"recognition":[59],"precise":[61,223],"understanding":[62],"these":[64],"IGEs":[65,492],"is":[66,236,317,416,459],"instrumental,":[67],"serving":[68],"foundation":[71],"grounding,":[74],"which":[75,275,369,390],"facilitates":[77],"downstream":[78],"tasks,":[79],"including":[80,206,329,426],"automated":[81],"testing.":[83],"A":[84],"straightforward":[85],"test":[87,117,138],"generator":[88],"randomly":[91],"within":[92,319],"app\u2019s":[94],"3D":[95,278,300],"environment,":[96],"making":[97],"it":[98,235],"trapped":[99],"in":[100,105,151,197,277,312,448],"uninteractable":[101],"space":[102,473,502],"resulting":[104],"an":[106,320],"ineffective":[107],"inefficient":[109],"testing":[110,467,481,488,499],"process.":[111],"contrast,":[113],"more":[115,142,417,491,495],"intelligent":[116],"generator,":[118],"informed":[119],"by":[120,209,442,468],"locations":[123],"semantics":[125,406,453],"IGEs,":[127,456],"make":[129],"wiser":[130],"decisions":[131],"interaction":[133,297],"targets":[134],"orders,":[136],"forming":[137],"sequences":[139],"that":[140,414,486],"cover":[141],"functionalities":[143],"faster.":[144],"most":[146],"detection":[149,165,196,232,260,315,344,424],"approaches":[150,189],"SE":[152],"are":[153],"designed":[154],"for":[155,229,262,334,345,373,461],"2D":[156],"mobile":[157],"apps":[158,397],"typically":[160],"train":[161],"supervised":[163],"object":[164],"model":[166],"based":[167,352,436],"large-scale":[170],"manually-labeled":[171],"dataset,":[173],"usually":[174],"pre-defined":[177],"set":[178],"clickable":[180],"categories":[183],"like":[184],"buttons":[185],"spinners.":[187],"Such":[188],"hardly":[191],"be":[192],"applied":[193],"to":[194,201,238,245,286],"apps,":[199,265],"due":[200],"multitude":[203],"challenges":[205],"complexities":[207],"posed":[208],"open-vocabulary":[210],"heterogeneous":[212],"categories,":[214],"intricacies":[215],"context-sensitive":[217,256],"interactability,":[218],"necessities":[221],"spatial":[224,301,309],"perception":[225],"visual-semantic":[227],"alignment":[228],"results.":[233],"Thus,":[234],"necessary":[237],"embark":[239],"research":[243],"tailored":[244,285],"apps.":[247],"this":[249],"paper,":[250],"we":[251,384],"propose":[252],"first":[254,290,387],"zero-shot":[255],"framework":[261],"named":[266],"Orienter.":[267],"Rather":[268],"than":[269,419,498],"relying":[270],"generic":[272],"visual":[273],"grounding":[274],"fails":[276],"environments,":[279],"Orienter":[280,325,415,458],"introduces":[281],"structured":[283],"workflow":[284],"constraints.":[288],"It":[289],"synthesizes":[291],"XR-specific":[292],"contexts":[294,372],"(e.g.,":[295],"global":[296],"paradigms":[298],"properties)":[302],"before":[303],"performing":[304],"detection.":[305],"To":[306,376],"overcome":[307],"severe":[308],"hallucinations":[310],"inherent":[311],"LMMs,":[313],"process":[316],"iterated":[318],"XR-constrained":[321],"reflection":[322],"loop.":[323],"Specifically,":[324],"contains":[326,391],"three":[327],"components,":[328],"(1)":[330],"Semantic":[331],"context":[332],"comprehension":[333],"capturing":[335],"apps\u2019":[337],"context,":[339],"(2)":[340],"Reflection-directed":[341],"candidate":[343],"identifying":[346],"localizing":[348],"valid":[349],"multi-perspective":[354],"description":[355],"guided":[356],"detection,":[358],"well":[360],"feedback-directed":[362],"reflection,":[363],"(3)":[365],"Context-sensitive":[366],"interactability":[367,374],"classification":[368],"integrates":[370],"prediction.":[375],"evaluate":[377],"our":[378],"approach":[379],"facilitate":[381],"follow-up":[382],"research,":[383],"construct":[385],"benchmark":[388],"dataset":[389,412],"1,552":[392],"images":[393],"from":[394,474],"100":[395],"industrial-setting":[396],"Steam,":[399],"4,470":[401],"annotations":[403],"766":[405],"categories.":[407],"Extensive":[408],"experiments":[409],"demonstrate":[413,485],"effective":[418,496],"state-of-the-art":[421],"approaches,":[425],"general":[427],"GUI-automation-targeted":[429],"vision":[430],"language":[431],"models,":[432,437],"deep":[434],"learning":[435],"surpassing":[438],"their":[439],"F1":[440],"Score":[441],"at":[443],"least":[444],"34.9%":[445],"20.1\u00d7":[447],"distinguishing":[449],"interactibility":[451],"respectively.":[457],"beneficial":[460],"boosting":[462],"performance":[464],"automatic":[466],"isolating":[469],"action":[472,501],"whole":[476],"space,":[477],"regardless":[478],"strategies":[482],"employed.":[483],"Experiments":[484],"Orienter-guided":[487],"covers":[489],"103.1%":[490],"125.7%":[494],"interactions":[497],"without":[500],"isolation.":[503]},"counts_by_year":[],"updated_date":"2026-07-02T06:12:58.138171","created_date":"2026-07-01T00:00:00"}