{"id":"https://openalex.org/W4412888052","doi":"https://doi.org/10.18653/v1/2025.findings-acl.920","title":"JARVIS-VLA: Post-Training Large-Scale Vision Language Models to Play Visual Games with Keyboards and Mouse","display_name":"JARVIS-VLA: Post-Training Large-Scale Vision Language Models to Play Visual Games with Keyboards and Mouse","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888052","doi":"https://doi.org/10.18653/v1/2025.findings-acl.920"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.920","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.920","pdf_url":"https://aclanthology.org/2025.findings-acl.920.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.920.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003830679","display_name":"Muyao Li","orcid":"https://orcid.org/0009-0006-1532-4536"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Muyao Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100413085","display_name":"Zihao Wang","orcid":"https://orcid.org/0000-0002-3919-0396"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zihao Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101369119","display_name":"Kaichen He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kaichen He","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100689683","display_name":"Xiaojian Ma","orcid":"https://orcid.org/0000-0002-2611-6568"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaojian Ma","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112110952","display_name":"Yitao Liang","orcid":"https://orcid.org/0009-0009-0944-7814"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yitao Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"17878","last_page":"17899"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.9160000085830688,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.909600019454956,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6858362555503845},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5834900736808777},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5730466842651367},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5252870321273804},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4036281704902649},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.349927693605423},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.34223827719688416},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.3267544209957123},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.09350377321243286},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07751792669296265}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6858362555503845},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5834900736808777},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5730466842651367},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5252870321273804},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4036281704902649},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.349927693605423},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34223827719688416},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.3267544209957123},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.09350377321243286},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07751792669296265},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.920","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.920","pdf_url":"https://aclanthology.org/2025.findings-acl.920.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.920","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.920","pdf_url":"https://aclanthology.org/2025.findings-acl.920.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G453759813","display_name":null,"funder_award_id":"2022ZD0114902","funder_id":"https://openalex.org/F4320329860","funder_display_name":"National Science and Technology Major Project"}],"funders":[{"id":"https://openalex.org/F4320318547","display_name":"Baidu","ror":"https://ror.org/03vs3wt56"},{"id":"https://openalex.org/F4320329860","display_name":"National Science and Technology Major Project","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888052.pdf","grobid_xml":"https://content.openalex.org/works/W4412888052.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W4412817058","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W4366179611"],"abstract_inverted_index":{"Recently,":[0],"action-based":[1],"decision-making":[2,23],"in":[3,22,87,116,166],"open-world":[4],"environments":[5],"has":[6,27],"gained":[7],"significant":[8,145],"attention.Visual":[9],"Language":[10,47],"Action":[11],"(VLA)":[12],"models,":[13,175],"pretrained":[14],"on":[15,30,67,104,123,153],"large-scale":[16],"web":[17],"datasets,":[18],"have":[19,171],"shown":[20],"promise":[21],"tasks.However,":[24],"previous":[25],"work":[26],"primarily":[28],"focused":[29],"action":[31,99],"post-training,":[32],"often":[33],"neglecting":[34],"enhancements":[35],"to":[36,60,143,178],"the":[37,56,84,112,149,173],"foundation":[38,57],"model":[39,58,115],"itself.In":[40],"response,":[41],"we":[42,109],"introduce":[43],"Act":[44],"from":[45],"Visual":[46],"Post-Training":[48],"(ActVLP),":[49],"a":[50,68,144,154],"novel":[51],"training":[52],"paradigm.ActVLP":[53],"distinctively":[54],"enhances":[55],"prior":[59],"action-specific":[61],"tuning":[62],"by":[63],"first":[64,113],"post-training":[65],"it":[66],"curated":[69],"set":[70,156],"of":[71,157],"environment-specific":[72],"visual":[73,90],"and":[74,92,134,176],"linguistic":[75],"tasks":[76],"using":[77],"self-supervised":[78],"learning.This":[79],"initial":[80],"stage":[81],"significantly":[82],"improves":[83],"model's":[85],"capabilities":[86],"world":[88],"knowledge,":[89],"recognition,":[91],"spatial":[93],"grounding.Subsequently,":[94],"this":[95,107],"strengthened":[96],"VLM":[97],"undergoes":[98],"posttraining":[100],"via":[101],"imitation":[102,163],"learning":[103],"trajectory":[105],"datasets.Following":[106],"paradigm,":[108],"develop":[110],"JARVIS-VLA,":[111],"VLA":[114],"Minecraft":[117],"that":[118,138],"can":[119,184],"follow":[120],"human":[121],"instructions":[122],"over":[124,148],"1k":[125],"different":[126],"atomic":[127,158],"tasks,":[128],"including":[129],"crafting,":[130],"smelting,":[131],"cooking,":[132],"mining,":[133],"killing.Our":[135],"experiments":[136],"demonstrate":[137],"our":[139],"ActVLP":[140],"paradigm":[141],"leads":[142],"40%":[146],"improvement":[147],"best":[150],"agent":[151],"baseline":[152],"diverse":[155],"tasks.Furthermore,":[159],"JARVIS-VLA":[160],"surpasses":[161],"traditional":[162],"learning-based":[164],"policies":[165],"Minecraft,":[167],"achieving":[168],"state-of-the-art":[169],"performance.We":[170],"open-sourced":[172],"code,":[174],"datasets":[177],"foster":[179],"further":[180],"research.The":[181],"project":[182],"page":[183],"be":[185],"found":[186],"at":[187],"https://craftjarvis.":[188],"github.io/JarvisVLA.":[189]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}