{"id":"https://openalex.org/W7161765557","doi":"https://doi.org/10.48550/arxiv.2605.18758","title":"OmniGUI: Benchmarking GUI Agents in Omni-Modal Smartphone Environments","display_name":"OmniGUI: Benchmarking GUI Agents in Omni-Modal Smartphone Environments","publication_year":2026,"publication_date":"2026-04-03","ids":{"openalex":"https://openalex.org/W7161765557","doi":"https://doi.org/10.48550/arxiv.2605.18758"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.18758","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18758","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.18758","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027444024","display_name":"F\u00e9lix Henry","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Henry, Felix","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136598176","display_name":"Xiaochen Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Xiaochen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052390848","display_name":"Jiangyou Zhu","orcid":"https://orcid.org/0000-0003-1512-6784"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Jiangyou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136539009","display_name":"Yangfan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yangfan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136590723","display_name":"Bingqian Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Bingqian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136605717","display_name":"Min Chen","orcid":"https://orcid.org/0000-0002-8774-4074"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Min","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5106404139","display_name":"Shiyu Huang","orcid":"https://orcid.org/0009-0008-9541-0622"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Shiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5297999978065491,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5297999978065491,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.09650000184774399,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.05310000106692314,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6661999821662903},{"id":"https://openalex.org/keywords/graphical-user-interface","display_name":"Graphical user interface","score":0.6420000195503235},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.5582000017166138},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4925999939441681},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.4779999852180481},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.47589999437332153},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.42399999499320984},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.40549999475479126},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.3912999927997589}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7886999845504761},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6661999821662903},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.6420000195503235},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.598800003528595},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5582000017166138},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4925999939441681},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4779999852180481},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.47589999437332153},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.42399999499320984},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.3912999927997589},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.35120001435279846},{"id":"https://openalex.org/C155846161","wikidata":"https://www.wikidata.org/wiki/Q1143367","display_name":"Graphical model","level":2,"score":0.3476000130176544},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.34540000557899475},{"id":"https://openalex.org/C2780799671","wikidata":"https://www.wikidata.org/wiki/Q17087362","display_name":"Transient (computer programming)","level":2,"score":0.3440000116825104},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.32019999623298645},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3086000084877014},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.30469998717308044},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.29750001430511475},{"id":"https://openalex.org/C177606310","wikidata":"https://www.wikidata.org/wiki/Q5674297","display_name":"Adaptability","level":2,"score":0.28200000524520874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27309998869895935},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2596000134944916},{"id":"https://openalex.org/C179254644","wikidata":"https://www.wikidata.org/wiki/Q13222844","display_name":"Moment (physics)","level":2,"score":0.25850000977516174},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2574999928474426},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.25429999828338623}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.18758","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18758","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.18758","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18758","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Current":[0],"benchmarks":[1],"for":[2,123],"graphical":[3],"user":[4],"interface":[5],"(GUI)":[6],"agents":[7,19,53],"predominantly":[8],"rely":[9],"on":[10,137],"static":[11,65,139],"screenshots.":[12],"However,":[13],"real-world":[14],"smartphone":[15,56],"interaction":[16],"routinely":[17],"requires":[18],"to":[20,50,118],"process":[21],"transient":[22],"audio":[23],"cues":[24],"and":[25,69,152,175],"temporal":[26,151],"video":[27,70],"dynamics":[28],"that":[29,131],"are":[30,101,178],"tightly":[31],"coupled":[32],"with":[33,90],"the":[34,45,181],"moment":[35],"of":[36,113],"action.":[37],"To":[38],"bridge":[39],"this":[40],"gap,":[41],"we":[42,107],"introduce":[43],"OmniGUI,":[44],"first":[46],"step-level":[47],"benchmark":[48],"designed":[49],"evaluate":[51],"GUI":[52,98],"in":[54,103,147,180],"omni-modal":[55,97,110],"environments.":[57],"OmniGUI":[58],"provides":[59],"continuous,":[60],"interleaved":[61,116],"multimodal":[62,92],"inputs":[63,117],"comprising":[64],"images,":[66],"synchronous":[67,150],"audio,":[68],"clips":[71],"at":[72],"every":[73],"action":[74,83,142],"step.":[75],"The":[76,170],"dataset":[77],"encompasses":[78],"709":[79],"expert-demonstrated":[80],"episodes":[81],"(2,579":[82],"steps)":[84],"across":[85],"29":[86],"applications,":[87],"systematically":[88],"annotated":[89],"objective":[91],"dependency":[93],"levels.":[94],"Because":[95],"dedicated":[96],"agent":[99,121],"frameworks":[100],"currently":[102],"their":[104,141],"nascent":[105],"stage,":[106],"select":[108],"foundational":[109],"models":[111,134],"capable":[112],"natively":[114],"processing":[115,166],"serve":[119],"as":[120],"proxies":[122],"our":[124],"initial":[125],"baselines.":[126],"Our":[127],"empirical":[128],"evaluation":[129,173],"reveals":[130],"while":[132],"current":[133],"exhibit":[135],"competency":[136],"visually":[138],"tasks,":[140],"prediction":[143],"performance":[144],"degrades":[145],"significantly":[146],"environments":[148],"requiring":[149],"auditory":[153],"signals.":[154],"Furthermore,":[155],"ablation":[156],"studies":[157],"isolate":[158],"specific":[159],"operational":[160],"bottlenecks,":[161],"notably":[162],"cross-modal":[163],"interference":[164],"when":[165],"task-irrelevant":[167],"environmental":[168],"noise.":[169],"complete":[171],"dataset,":[172],"pipeline,":[174],"baseline":[176],"prompts":[177],"provided":[179],"supplementary":[182],"material.":[183],"Project":[184],"page:":[185],"https://omni-gui.github.io.":[186]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-21T00:00:00"}