{"id":"https://openalex.org/W4402727710","doi":"https://doi.org/10.1109/cvpr52733.2024.01262","title":"AssistGUI: Task-Oriented PC Graphical User Interface Automation","display_name":"AssistGUI: Task-Oriented PC Graphical User Interface Automation","publication_year":2024,"publication_date":"2024-06-16","ids":{"openalex":"https://openalex.org/W4402727710","doi":"https://doi.org/10.1109/cvpr52733.2024.01262"},"language":"en","primary_location":{"id":"doi:10.1109/cvpr52733.2024.01262","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52733.2024.01262","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001133932","display_name":"Difei Gao","orcid":"https://orcid.org/0000-0001-8494-3492"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Difei Gao","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100754485","display_name":"Lei Ji","orcid":"https://orcid.org/0000-0002-7569-3265"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Lei Ji","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037364812","display_name":"Zechen Bai","orcid":"https://orcid.org/0000-0001-5953-0442"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Zechen Bai","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109786811","display_name":"Mingyu Ouyang","orcid":null},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Mingyu Ouyang","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043375842","display_name":"Peiran Li","orcid":"https://orcid.org/0000-0002-2426-9763"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Peiran Li","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113395524","display_name":"Donzxing Mao","orcid":null},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Donzxing Mao","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104048097","display_name":"Qinchen Wu","orcid":"https://orcid.org/0009-0005-6784-8297"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Qinchen Wu","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057762021","display_name":"Weichen Zhang","orcid":"https://orcid.org/0000-0003-2929-0921"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Weichen Zhang","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058808692","display_name":"Peiyi Wang","orcid":"https://orcid.org/0000-0002-9904-0664"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Peiyi Wang","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104333305","display_name":"Xiangwu Guo","orcid":null},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xiangwu Guo","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059608278","display_name":"Hengxu Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Hengxu Wang","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084879213","display_name":"Luowei Zhou","orcid":"https://orcid.org/0000-0003-1197-0101"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Luowei Zhou","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068937750","display_name":"Mike Zheng Shou","orcid":"https://orcid.org/0000-0002-7681-2166"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Mike Zheng Shou","raw_affiliation_strings":["National University of Singapore,Show Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore,Show Lab","institution_ids":["https://openalex.org/I165932596"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I165932596"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"13289","last_page":"13298"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9853000044822693,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9853000044822693,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.9616000056266785,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10888","display_name":"Augmented Reality Applications","score":0.9545000195503235,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7934306263923645},{"id":"https://openalex.org/keywords/graphical-user-interface","display_name":"Graphical user interface","score":0.7147320508956909},{"id":"https://openalex.org/keywords/graphical-user-interface-testing","display_name":"Graphical user interface testing","score":0.6686001420021057},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6493986248970032},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.6160812973976135},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5709962844848633},{"id":"https://openalex.org/keywords/multiple-document-interface","display_name":"Multiple document interface","score":0.5668785572052002},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.5054476261138916},{"id":"https://openalex.org/keywords/10-foot-user-interface","display_name":"10-foot user interface","score":0.4742278754711151},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.4595784842967987},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.4434130787849426},{"id":"https://openalex.org/keywords/user-interface-design","display_name":"User interface design","score":0.3751268684864044},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.3392472267150879},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10211655497550964},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.09607020020484924}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7934306263923645},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.7147320508956909},{"id":"https://openalex.org/C66153210","wikidata":"https://www.wikidata.org/wiki/Q5597182","display_name":"Graphical user interface testing","level":4,"score":0.6686001420021057},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6493986248970032},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.6160812973976135},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5709962844848633},{"id":"https://openalex.org/C125367622","wikidata":"https://www.wikidata.org/wiki/Q645154","display_name":"Multiple document interface","level":5,"score":0.5668785572052002},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.5054476261138916},{"id":"https://openalex.org/C197070257","wikidata":"https://www.wikidata.org/wiki/Q4545902","display_name":"10-foot user interface","level":4,"score":0.4742278754711151},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.4595784842967987},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.4434130787849426},{"id":"https://openalex.org/C149229913","wikidata":"https://www.wikidata.org/wiki/Q135707","display_name":"User interface design","level":3,"score":0.3751268684864044},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3392472267150879},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10211655497550964},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.09607020020484924},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cvpr52733.2024.01262","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cvpr52733.2024.01262","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320698","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":78,"referenced_works":["https://openalex.org/W2889037901","https://openalex.org/W2963037989","https://openalex.org/W2963545046","https://openalex.org/W3034392229","https://openalex.org/W3116323972","https://openalex.org/W3157199281","https://openalex.org/W3164797848","https://openalex.org/W3191804539","https://openalex.org/W4221143046","https://openalex.org/W4281735693","https://openalex.org/W4282984144","https://openalex.org/W4283828996","https://openalex.org/W4304195432","https://openalex.org/W4312516512","https://openalex.org/W4320165837","https://openalex.org/W4322718191","https://openalex.org/W4323717348","https://openalex.org/W4324128075","https://openalex.org/W4327810433","https://openalex.org/W4353113046","https://openalex.org/W4361866031","https://openalex.org/W4361866080","https://openalex.org/W4366233310","https://openalex.org/W4366566341","https://openalex.org/W4379255871","https://openalex.org/W4380374951","https://openalex.org/W4380993798","https://openalex.org/W4384918194","https://openalex.org/W4384918448","https://openalex.org/W4385327683","https://openalex.org/W4385473486","https://openalex.org/W4385574233","https://openalex.org/W4385574370","https://openalex.org/W4385682136","https://openalex.org/W4386302825","https://openalex.org/W4386976823","https://openalex.org/W4387559318","https://openalex.org/W4387800098","https://openalex.org/W4388685755","https://openalex.org/W4389520250","https://openalex.org/W4390872747","https://openalex.org/W4390874575","https://openalex.org/W4402670435","https://openalex.org/W6741753925","https://openalex.org/W6749028843","https://openalex.org/W6757247981","https://openalex.org/W6794385437","https://openalex.org/W6795479624","https://openalex.org/W6809646742","https://openalex.org/W6811027297","https://openalex.org/W6838792323","https://openalex.org/W6839827046","https://openalex.org/W6846178434","https://openalex.org/W6846254642","https://openalex.org/W6848749110","https://openalex.org/W6849732303","https://openalex.org/W6850071225","https://openalex.org/W6850140740","https://openalex.org/W6850625674","https://openalex.org/W6850764995","https://openalex.org/W6850787431","https://openalex.org/W6850886341","https://openalex.org/W6850982024","https://openalex.org/W6851513886","https://openalex.org/W6851958866","https://openalex.org/W6852410717","https://openalex.org/W6853249947","https://openalex.org/W6853453387","https://openalex.org/W6853930081","https://openalex.org/W6854586193","https://openalex.org/W6854738657","https://openalex.org/W6854818647","https://openalex.org/W6854866820","https://openalex.org/W6856086692","https://openalex.org/W6856160515","https://openalex.org/W6857608137","https://openalex.org/W6857660107","https://openalex.org/W6858025407"],"related_works":["https://openalex.org/W3180179400","https://openalex.org/W1539807119","https://openalex.org/W2197102099","https://openalex.org/W2367556405","https://openalex.org/W2369004040","https://openalex.org/W963721989","https://openalex.org/W2188599776","https://openalex.org/W2659420435","https://openalex.org/W345736368","https://openalex.org/W2354564824"],"abstract_inverted_index":{"Graphical":[0],"User":[1],"Interface":[2],"(GUI)":[3],"automation":[4],"holds":[5],"significant":[6],"promise":[7],"for":[8,108,178],"assisting":[9],"users":[10],"with":[11,151,166],"complex":[12],"tasks,":[13],"thereby":[14],"boosting":[15],"human":[16],"productivity.":[17],"Existing":[18],"works":[19],"leveraging":[20],"Large":[21],"Language":[22],"Model":[23],"(LLM)":[24],"or":[25],"LLM-based":[26],"AI":[27],"agents":[28,121],"have":[29],"shown":[30],"capabilities":[31],"in":[32,76,144,181],"automating":[33],"tasks":[34,42,88],"on":[35,72,161],"Android":[36],"and":[37,50,70,98,130],"Web":[38],"platforms.":[39],"However,":[40],"these":[41],"are":[43,64],"primarily":[44],"aimed":[45],"at":[46],"simple":[47],"device":[48],"usage":[49],"entertainment":[51],"operations.":[52],"This":[53],"paper":[54],"presents":[55],"a":[56,84,114,157,167],"novel":[57],"benchmark,":[58],"Assistgui,":[59],"to":[60,78,122],"evaluate":[61],"whether":[62],"models":[63],"capable":[65],"of":[66,86,170],"manipulating":[67],"the":[68,73,104,147,152,171,176],"mouse":[69],"keyboard":[71],"Windows":[74],"platform":[75],"response":[77],"user-requested":[79],"tasks.":[80],"We":[81,164],"carefully":[82],"collected":[83],"set":[85],"100":[87],"from":[89],"nine":[90],"widely-used":[91],"software":[92],"applications,":[93],"such":[94],"as,":[95],"After":[96],"Effects":[97],"MS":[99],"Word,":[100],"each":[101],"accompanied":[102],"by":[103],"necessary":[105],"project":[106],"files":[107],"better":[109],"evaluation.":[110],"Moreover,":[111],"we":[112],"propose":[113],"multi-agent":[115,138],"collaboration":[116,139],"framework,":[117],"which":[118],"incorporates":[119],"four":[120],"perform":[123],"task":[124],"decomposition,":[125],"GUI":[126],"parsing,":[127],"action":[128],"generation,":[129],"reflection.":[131],"Our":[132],"experimental":[133],"results":[134],"reveal":[135],"that":[136],"our":[137,162],"mechanism":[140],"outshines":[141],"existing":[142],"methods":[143],"performance.":[145],"Nevertheless,":[146],"potential":[148],"remains":[149],"substantial,":[150],"best":[153],"model":[154],"attaining":[155],"only":[156],"46%":[158],"success":[159],"rate":[160],"benchmark.":[163],"conclude":[165],"thorough":[168],"analysis":[169],"current":[172],"methods'":[173],"limitations,":[174],"setting":[175],"stage":[177],"future":[179],"breakthroughs":[180],"this":[182],"domain.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}