{"id":"https://openalex.org/W4403333948","doi":"https://doi.org/10.1145/3654777.3676386","title":"VisionTasker: Mobile Task Automation Using Vision Based UI Understanding and LLM Task Planning","display_name":"VisionTasker: Mobile Task Automation Using Vision Based UI Understanding and LLM Task Planning","publication_year":2024,"publication_date":"2024-10-11","ids":{"openalex":"https://openalex.org/W4403333948","doi":"https://doi.org/10.1145/3654777.3676386"},"language":"en","primary_location":{"id":"doi:10.1145/3654777.3676386","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3654777.3676386","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 37th Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019408798","display_name":"Yunpeng Song","orcid":"https://orcid.org/0000-0002-4186-0408"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunpeng Song","raw_affiliation_strings":["MOE KLINNS Lab, Xi'an Jiaotong University, China"],"raw_orcid":"https://orcid.org/0000-0002-4186-0408","affiliations":[{"raw_affiliation_string":"MOE KLINNS Lab, Xi'an Jiaotong University, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044143625","display_name":"Yiheng Bian","orcid":"https://orcid.org/0009-0004-8628-1751"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiheng Bian","raw_affiliation_strings":["MOE KLINNS Lab, Xi'an Jiaotong University, China"],"raw_orcid":"https://orcid.org/0009-0004-8628-1751","affiliations":[{"raw_affiliation_string":"MOE KLINNS Lab, Xi'an Jiaotong University, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074559140","display_name":"Yongtao Tang","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongtao Tang","raw_affiliation_strings":["MOE KLINNS Lab, Xi'an Jiaotong University, China"],"raw_orcid":"https://orcid.org/0009-0007-9941-8456","affiliations":[{"raw_affiliation_string":"MOE KLINNS Lab, Xi'an Jiaotong University, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109026141","display_name":"Guiyu Ma","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guiyu Ma","raw_affiliation_strings":["MOE KLINNS Lab, Xi'an Jiaotong University, China"],"raw_orcid":"https://orcid.org/0009-0000-9910-5301","affiliations":[{"raw_affiliation_string":"MOE KLINNS Lab, Xi'an Jiaotong University, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082529620","display_name":"Zhongmin Cai","orcid":"https://orcid.org/0000-0002-4903-3992"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhongmin Cai","raw_affiliation_strings":["MOE KLINNS Lab, Xi'an Jiaotong University, China"],"raw_orcid":"https://orcid.org/0000-0002-4903-3992","affiliations":[{"raw_affiliation_string":"MOE KLINNS Lab, Xi'an Jiaotong University, China","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I87445476"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9861999750137329,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7590583562850952},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7584601640701294},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.6577447652816772},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.6554109454154968},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.44246238470077515},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.19718420505523682},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1509179174900055}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7590583562850952},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7584601640701294},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6577447652816772},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.6554109454154968},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.44246238470077515},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.19718420505523682},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1509179174900055},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3654777.3676386","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3654777.3676386","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 37th Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W1748744376","https://openalex.org/W1982451263","https://openalex.org/W2127141656","https://openalex.org/W2765874585","https://openalex.org/W2896625930","https://openalex.org/W2914397179","https://openalex.org/W2914783728","https://openalex.org/W2997864923","https://openalex.org/W3010331969","https://openalex.org/W3034383590","https://openalex.org/W3048516901","https://openalex.org/W3100314869","https://openalex.org/W3101041928","https://openalex.org/W3116323972","https://openalex.org/W3135367836","https://openalex.org/W3160022552","https://openalex.org/W3160042174","https://openalex.org/W3163374010","https://openalex.org/W3194302499","https://openalex.org/W3199730823","https://openalex.org/W3205557064","https://openalex.org/W3206316161","https://openalex.org/W4221143046","https://openalex.org/W4224308101","https://openalex.org/W4224982588","https://openalex.org/W4224988754","https://openalex.org/W4225089191","https://openalex.org/W4229030834","https://openalex.org/W4280501884","https://openalex.org/W4283320615","https://openalex.org/W4285294723","https://openalex.org/W4285605356","https://openalex.org/W4297220110","https://openalex.org/W4298187317","https://openalex.org/W4312516512","https://openalex.org/W4361759487","https://openalex.org/W4366549767","https://openalex.org/W4384155512","https://openalex.org/W4391558516","https://openalex.org/W4396833734","https://openalex.org/W4402753874","https://openalex.org/W6778883912","https://openalex.org/W6810081322"],"related_works":["https://openalex.org/W3196817267","https://openalex.org/W1976600725","https://openalex.org/W1564680838","https://openalex.org/W2003125260","https://openalex.org/W2060591604","https://openalex.org/W1992291644","https://openalex.org/W2166791242","https://openalex.org/W2585162246","https://openalex.org/W1934413089","https://openalex.org/W2098419343"],"abstract_inverted_index":{"Mobile":[0],"task":[1,68,100,104,137],"automation":[2,105],"is":[3,214],"an":[4,189],"emerging":[5],"field":[6],"that":[7,170],"leverages":[8],"AI":[9],"to":[10,38,46,56,66,146],"streamline":[11],"and":[12,25,44,60,76,98,158,165,203,216],"optimize":[13],"the":[14,53,127,147,156,160,210],"execution":[15],"of":[16],"routine":[17],"tasks":[18,43,187,198],"on":[19,41,188],"mobile":[20,103],"devices,":[21],"thereby":[22],"enhancing":[23,163],"efficiency":[24],"productivity.":[26],"Traditional":[27],"methods,":[28,174],"such":[29],"as":[30],"Programming":[31],"By":[32],"Demonstration":[33],"(PBD),":[34],"are":[35],"limited":[36],"due":[37],"their":[39],"dependence":[40],"predefined":[42],"susceptibility":[45],"app":[47],"updates.":[48],"Recent":[49],"advancements":[50],"have":[51,73],"utilized":[52],"view":[54,71,130],"hierarchy":[55],"collect":[57],"UI":[58,96,114,123,177],"information":[59],"employed":[61],"Large":[62],"Language":[63],"Models":[64],"(LLM)":[65],"enhance":[67],"automation.":[69],"However,":[70],"hierarchies":[72],"accessibility":[74],"issues":[75],"face":[77],"potential":[78],"problems":[79],"like":[80],"missing":[81],"object":[82],"descriptions":[83],"or":[84],"misaligned":[85],"structures.":[86],"This":[87],"paper":[88],"introduces":[89],"VisionTasker,":[90],"a":[91,107,113,121,135,144],"two-stage":[92],"framework":[93],"combining":[94],"vision-based":[95,122],"understanding":[97,124],"LLM":[99,150],"planning,":[101],"for":[102,129],"in":[106,183,197],"step-by-step":[108,136],"manner.":[109],"VisionTasker":[110,171,192,213],"firstly":[111],"converts":[112],"screenshot":[115],"into":[116],"natural":[117],"language":[118],"interpretations":[119],"using":[120],"approach,":[125],"eliminating":[126],"need":[128],"hierarchies.":[131],"Secondly,":[132],"it":[133],"adopts":[134],"planning":[138],"method,":[139],"presenting":[140],"one":[141],"interface":[142,157],"at":[143,218],"time":[145],"LLM.":[148],"The":[149],"then":[151],"identifies":[152],"relevant":[153],"elements":[154],"within":[155],"determines":[159],"next":[161],"action,":[162],"accuracy":[164],"practicality.":[166],"Extensive":[167],"experiments":[168],"show":[169,201],"outperforms":[172],"previous":[173],"providing":[175],"effective":[176],"representations":[178],"across":[179],"four":[180],"datasets.":[181],"Additionally,":[182],"automating":[184],"147":[185],"real-world":[186],"Android":[190],"smartphone,":[191],"demonstrates":[193],"advantages":[194],"over":[195],"humans":[196,200],"where":[199],"unfamiliarity":[202],"shows":[204],"significant":[205],"improvements":[206],"when":[207],"integrated":[208],"with":[209],"PBD":[211],"mechanism.":[212],"open-source":[215],"available":[217],"https://github.com/AkimotoAyako/VisionTasker.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":10}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}