{"id":"https://openalex.org/W7153328783","doi":"https://doi.org/10.48550/arxiv.2604.07831","title":"Are GUI Agents Focused Enough? Automated Distraction via Semantic-level UI Element Injection","display_name":"Are GUI Agents Focused Enough? Automated Distraction via Semantic-level UI Element Injection","publication_year":2026,"publication_date":"2026-04-09","ids":{"openalex":"https://openalex.org/W7153328783","doi":"https://doi.org/10.48550/arxiv.2604.07831"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.07831","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07831","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.07831","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104278895","display_name":"Wenkui Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Wenkui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071409132","display_name":"Chao Jin","orcid":"https://orcid.org/0000-0002-4902-6485"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133343484","display_name":"Haisu Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Haisu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051273658","display_name":"Weilin Luo","orcid":"https://orcid.org/0000-0002-3733-9361"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Weilin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133389903","display_name":"Derek Yuen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuen, Derek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133387011","display_name":"Kun Shao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shao, Kun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133362916","display_name":"Junxian Duan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duan, Junxian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133387578","display_name":"Huaibo Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Huaibo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133367169","display_name":"Jie Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Jie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133371100","display_name":"Ran He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Ran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.691100001335144,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.691100001335144,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.10109999775886536,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.1005999967455864,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5382999777793884},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.5196999907493591},{"id":"https://openalex.org/keywords/overlay","display_name":"Overlay","score":0.5148000121116638},{"id":"https://openalex.org/keywords/element","display_name":"Element (criminal law)","score":0.5012999773025513},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.48339998722076416},{"id":"https://openalex.org/keywords/distraction","display_name":"Distraction","score":0.4237000048160553},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.4088999927043915}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6349999904632568},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5382999777793884},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5196999907493591},{"id":"https://openalex.org/C136085584","wikidata":"https://www.wikidata.org/wiki/Q910289","display_name":"Overlay","level":2,"score":0.5148000121116638},{"id":"https://openalex.org/C200288055","wikidata":"https://www.wikidata.org/wiki/Q2621792","display_name":"Element (criminal law)","level":2,"score":0.5012999773025513},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.48339998722076416},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4593999981880188},{"id":"https://openalex.org/C2776378700","wikidata":"https://www.wikidata.org/wiki/Q3030775","display_name":"Distraction","level":2,"score":0.4237000048160553},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4088999927043915},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.37779998779296875},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36820000410079956},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34360000491142273},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3084000051021576},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3010999858379364},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.289000004529953},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.27869999408721924},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C135628077","wikidata":"https://www.wikidata.org/wiki/Q220184","display_name":"Finite element method","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C168820333","wikidata":"https://www.wikidata.org/wiki/Q448889","display_name":"Visual inspection","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2547999918460846},{"id":"https://openalex.org/C177606310","wikidata":"https://www.wikidata.org/wiki/Q5674297","display_name":"Adaptability","level":2,"score":0.2500999867916107}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.07831","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07831","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.07831","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07831","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.7704178094863892}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Existing":[0],"red-teaming":[1,46],"studies":[2],"on":[3,89,110],"GUI":[4],"agents":[5],"face":[6],"two":[7],"fundamental":[8],"limitations:":[9],"adversarial":[10],"perturbations":[11],"require":[12],"white-box":[13],"access":[14],"unavailable":[15],"in":[16,136],"commercial":[17],"deployments,":[18],"while":[19],"prompt":[20,86],"injection":[21,108],"is":[22],"increasingly":[23],"neutralized":[24],"by":[25],"stronger":[26],"safety":[27],"alignment.":[28],"To":[29],"study":[30],"robustness":[31],"under":[32],"a":[33,44,66],"more":[34],"practical":[35],"threat":[36],"model,":[37],"we":[38],"propose":[39],"Semantic-level":[40],"UI":[41,53],"Element":[42],"Injection,":[43],"black-box":[45],"paradigm":[47],"that":[48,73,102,150,158],"overlays":[49],"safety-aligned":[50],"and":[51,83,115],"harmless":[52],"elements":[54],"onto":[55],"screenshots":[56],"to":[57],"misdirect":[58],"the":[59,79,111,125,129,133],"agent's":[60],"visual":[61],"grounding.":[62],"Our":[63],"method":[64],"couples":[65],"modular":[67],"Editor--Overlapper--Victim":[68],"pipeline":[69],"with":[70],"iterative":[71],"search":[72],"samples":[74],"multiple":[75],"candidate":[76],"edits,":[77],"keeps":[78],"best":[80],"cumulative":[81],"overlay,":[82],"adapts":[84],"future":[85],"strategies":[87],"based":[88],"previous":[90],"failures.":[91],"Experiments":[92],"across":[93,118],"19":[94],"victim":[95,130],"models":[96],"spanning":[97],"8":[98],"model":[99],"families":[100],"show":[101],"strategic":[103],"optimization":[104],"substantially":[105],"outperforms":[106],"random":[107,147],"(3.5-6.9x":[109],"most":[112],"robust":[113],"victims)":[114],"transfers":[116],"near-perfectly":[117],"architectures,":[119],"confirming":[120],"model-agnostic":[121],"visual-semantic":[122],"vulnerabilities.":[123],"After":[124],"first":[126],"successful":[127],"attack,":[128],"still":[131],"clicks":[132],"attacker-controlled":[134],"icon":[135],"over":[137],"15\\%":[138],"of":[139],"subsequent":[140],"independent":[141],"trials":[142],"versus":[143],"below":[144],"1%":[145],"for":[146],"injection,":[148],"establishing":[149],"strategically":[151],"placed":[152],"icons":[153],"act":[154],"as":[155],"persistent":[156],"attractors":[157],"causally":[159],"redirect":[160],"grounding":[161],"rather":[162],"than":[163],"introducing":[164],"incidental":[165],"clutter.":[166]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-11T00:00:00"}