{"id":"https://openalex.org/W4404988784","doi":"https://doi.org/10.48550/arxiv.2411.17465","title":"ShowUI: One Vision-Language-Action Model for GUI Visual Agent","display_name":"ShowUI: One Vision-Language-Action Model for GUI Visual Agent","publication_year":2024,"publication_date":"2024-11-26","ids":{"openalex":"https://openalex.org/W4404988784","doi":"https://doi.org/10.48550/arxiv.2411.17465"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2411.17465","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.17465","pdf_url":"https://arxiv.org/pdf/2411.17465","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.17465","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083481253","display_name":"Kevin Qinghong Lin","orcid":"https://orcid.org/0000-0003-2568-2346"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Lin, Kevin Qinghong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110673725","display_name":"Linjie Li","orcid":"https://orcid.org/0000-0002-9817-1793"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Linjie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001133932","display_name":"Difei Gao","orcid":"https://orcid.org/0000-0001-8494-3492"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Difei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050209478","display_name":"Zhengyuan Yang","orcid":"https://orcid.org/0000-0002-5808-0889"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Zhengyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101934104","display_name":"Shiwei Wu","orcid":"https://orcid.org/0000-0002-3206-6827"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Shiwei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037364812","display_name":"Zechen Bai","orcid":"https://orcid.org/0000-0001-5953-0442"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bai, Zechen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039070189","display_name":"Weixian Lei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lei, Weixian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101406479","display_name":"Lijuan Wang","orcid":"https://orcid.org/0000-0001-9813-9075"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Lijuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5068937750","display_name":"Mike Zheng Shou","orcid":"https://orcid.org/0000-0002-7681-2166"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shou, Mike Zheng","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5083481253"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.7086628675460815},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6453050374984741},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4668327867984772},{"id":"https://openalex.org/keywords/visual-language","display_name":"Visual language","score":0.4359932243824005},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41200146079063416},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4060613214969635},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.3895612955093384},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.37478625774383545},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.24418538808822632},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.22676756978034973},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.06508946418762207}],"concepts":[{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.7086628675460815},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6453050374984741},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4668327867984772},{"id":"https://openalex.org/C2780878386","wikidata":"https://www.wikidata.org/wiki/Q1659648","display_name":"Visual language","level":2,"score":0.4359932243824005},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41200146079063416},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4060613214969635},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3895612955093384},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.37478625774383545},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.24418538808822632},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.22676756978034973},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.06508946418762207},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2411.17465","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.17465","pdf_url":"https://arxiv.org/pdf/2411.17465","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"doi:10.48550/arxiv.2411.17465","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.17465","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.17465","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.17465","pdf_url":"https://arxiv.org/pdf/2411.17465","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4231937131","https://openalex.org/W2369501117","https://openalex.org/W323219885","https://openalex.org/W2063928587","https://openalex.org/W1487966966","https://openalex.org/W1589342014","https://openalex.org/W1480341462","https://openalex.org/W4246664336","https://openalex.org/W1924502949","https://openalex.org/W2141293686"],"abstract_inverted_index":{"Building":[0],"Graphical":[1],"User":[2],"Interface":[3],"(GUI)":[4],"assistants":[5],"holds":[6],"significant":[7,147],"promise":[8],"for":[9,44,93],"enhancing":[10],"human":[11],"workflow":[12],"productivity.":[13],"While":[14],"most":[15],"agents":[16],"are":[17,218],"language-based,":[18],"relying":[19],"on":[20],"closed-source":[21],"API":[22],"with":[23],"text-rich":[24],"meta-information":[25],"(e.g.,":[26],"HTML":[27],"or":[28,119],"accessibility":[29],"tree),":[30],"they":[31],"show":[32],"limitations":[33],"in":[34,56,117,167,211],"perceiving":[35],"UI":[36,80],"visuals":[37],"as":[38,78,90],"humans":[39],"do,":[40],"highlighting":[41],"the":[42,63,91,187,204],"need":[43],"GUI":[45,109,133,213],"visual":[46,180,214],"agents.":[47,215],"In":[48],"this":[49],"work,":[50],"we":[51],"develop":[52],"a":[53,142,155,163],"vision-language-action":[54],"model":[55,158,210],"digital":[57],"world,":[58],"namely":[59],"ShowUI,":[60,154],"which":[61],"features":[62],"following":[64],"innovations:":[65],"(i)":[66],"UI-Guided":[67],"Visual":[68],"Token":[69],"Selection":[70],"to":[71,126,145],"reduce":[72],"computational":[73],"costs":[74],"by":[75,136,189],"formulating":[76],"screenshots":[77],"an":[79],"connected":[81],"graph,":[82],"adaptively":[83],"identifying":[84],"their":[85],"redundant":[86,179],"relationship":[87],"and":[88,140,184,198,206],"serve":[89],"criteria":[92],"token":[94,173],"selection":[95,174],"during":[96,182],"self-attention":[97],"blocks;":[98],"(ii)":[99],"Interleaved":[100],"Vision-Language-Action":[101],"Streaming":[102],"that":[103],"flexibly":[104],"unifies":[105],"diverse":[106],"needs":[107],"within":[108],"tasks,":[110],"enabling":[111],"effective":[112],"management":[113],"of":[114,178,208],"visual-action":[115],"history":[116],"navigation":[118],"pairing":[120],"multi-turn":[121],"query-action":[122],"sequences":[123],"per":[124],"screenshot":[125,169],"enhance":[127],"training":[128,183],"efficiency;":[129],"(iii)":[130],"Small-scale":[131],"High-quality":[132],"Instruction-following":[134],"Datasets":[135],"careful":[137],"data":[138,148],"curation":[139],"employing":[141],"resampling":[143],"strategy":[144],"address":[146],"type":[149],"imbalances.":[150],"With":[151],"above":[152],"components,":[153],"lightweight":[156],"2B":[157],"using":[159],"256K":[160],"data,":[161],"achieves":[162],"strong":[164],"75.1%":[165],"accuracy":[166],"zero-shot":[168],"grounding.":[170],"Its":[171],"UI-guided":[172],"further":[175,202],"reduces":[176],"33%":[177],"tokens":[181],"speeds":[185],"up":[186],"performance":[188],"1.4x.":[190],"Navigation":[191],"experiments":[192],"across":[193],"web":[194],"Mind2Web,":[195],"mobile":[196],"AITW,":[197],"online":[199],"MiniWob":[200],"environments":[201],"underscore":[203],"effectiveness":[205],"potential":[207],"our":[209],"advancing":[212],"The":[216],"models":[217],"available":[219],"at":[220],"https://github.com/showlab/ShowUI.":[221]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}