{"id":"https://openalex.org/W4391833068","doi":"https://doi.org/10.48550/arxiv.2402.07945","title":"ScreenAgent: A Vision Language Model-driven Computer Control Agent","display_name":"ScreenAgent: A Vision Language Model-driven Computer Control Agent","publication_year":2024,"publication_date":"2024-02-09","ids":{"openalex":"https://openalex.org/W4391833068","doi":"https://doi.org/10.48550/arxiv.2402.07945"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.07945","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.07945","pdf_url":"https://arxiv.org/pdf/2402.07945","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.07945","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085758120","display_name":"Runliang Niu","orcid":"https://orcid.org/0000-0002-5361-030X"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Niu, Runliang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042031737","display_name":"Jindong Li","orcid":"https://orcid.org/0000-0002-4009-916X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jindong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100385190","display_name":"Shiqi Wang","orcid":"https://orcid.org/0009-0003-2011-3272"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shiqi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109085749","display_name":"Yali Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Yali","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113306709","display_name":"Xiyu Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Xiyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101304997","display_name":"Xueyuan Leng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leng, Xueyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032411105","display_name":"He Kong","orcid":"https://orcid.org/0000-0002-1382-4186"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kong, He","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029392006","display_name":"Yi Chang","orcid":"https://orcid.org/0000-0003-2697-8093"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Yi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5105840097","display_name":"Qi Wang","orcid":"https://orcid.org/0009-0004-4055-5278"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Qi","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5085758120"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.9297999739646912,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.9297999739646912,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6807804107666016},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5712775588035583},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43314042687416077},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.35461661219596863},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.34856653213500977},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.34029892086982727}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6807804107666016},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5712775588035583},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43314042687416077},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.35461661219596863},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34856653213500977},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34029892086982727}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.07945","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.07945","pdf_url":"https://arxiv.org/pdf/2402.07945","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.07945","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.07945","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.07945","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.07945","pdf_url":"https://arxiv.org/pdf/2402.07945","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391833068.pdf","grobid_xml":"https://content.openalex.org/works/W4391833068.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2755342338","https://openalex.org/W2058170566","https://openalex.org/W2036807459","https://openalex.org/W2772917594","https://openalex.org/W2775347418","https://openalex.org/W1969923398","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2079911747"],"abstract_inverted_index":{"Existing":[0],"Large":[1],"Language":[2,66],"Models":[3],"(LLM)":[4],"can":[5,41,82],"invoke":[6],"a":[7,32,44,64,73,139,148,174],"variety":[8,140],"of":[9,141],"tools":[10],"and":[11,23,85,95,109,121,134,159],"APIs":[12],"to":[13,48,70,115,157],"complete":[14,122],"complex":[15],"tasks.":[16,124,144],"The":[17,178],"computer,":[18,39],"as":[19],"the":[20,38,80,87,113,119,128],"most":[21],"powerful":[22],"universal":[24],"tool,":[25],"could":[26,168],"potentially":[27],"be":[28],"controlled":[29],"directly":[30],"by":[31,37,92],"trained":[33,147],"LLM":[34,176],"agent.":[35,177],"Powered":[36],"we":[40,59,126,146],"hopefully":[42],"build":[43],"more":[45,161],"generalized":[46],"agent":[47,69,81,114],"assist":[49],"humans":[50],"in":[51],"various":[52],"daily":[53,142],"digital":[54],"works.":[55],"In":[56],"this":[57,78],"paper,":[58],"construct":[60,127],"an":[61,101],"environment":[62,120],"for":[63],"Vision":[65],"Model":[67],"(VLM)":[68],"interact":[71,117],"with":[72,118],"real":[74],"computer":[75,143,153],"screen.":[76],"Within":[77],"environment,":[79],"observe":[83],"screenshots":[84,133],"manipulate":[86],"Graphics":[88],"User":[89],"Interface":[90],"(GUI)":[91],"outputting":[93],"mouse":[94],"keyboard":[96],"actions.":[97],"We":[98],"also":[99],"design":[100],"automated":[102],"control":[103,154],"pipeline":[104],"that":[105],"includes":[106],"planning,":[107],"acting,":[108],"reflecting":[110],"phases,":[111],"guiding":[112],"continuously":[116],"multi-step":[123],"Additionally,":[125],"ScreenAgent":[129],"Dataset,":[130],"which":[131,151],"collects":[132],"action":[135],"sequences":[136],"when":[137],"completing":[138],"Finally,":[145],"model,":[149],"ScreenAgent,":[150],"achieved":[152],"capabilities":[155],"comparable":[156],"GPT-4V":[158],"demonstrated":[160],"precise":[162],"UI":[163],"positioning":[164],"capabilities.":[165],"Our":[166],"attempts":[167],"inspire":[169],"further":[170],"research":[171],"on":[172],"building":[173],"generalist":[175],"code":[179],"is":[180],"available":[181],"at":[182],"\\url{https://github.com/niuzaisheng/ScreenAgent}.":[183]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}