{"id":"https://openalex.org/W4403752129","doi":"https://doi.org/10.48550/arxiv.2409.12889","title":"Can VLMs Play Action Role-Playing Games? Take Black Myth Wukong as a Study Case","display_name":"Can VLMs Play Action Role-Playing Games? Take Black Myth Wukong as a Study Case","publication_year":2024,"publication_date":"2024-09-19","ids":{"openalex":"https://openalex.org/W4403752129","doi":"https://doi.org/10.48550/arxiv.2409.12889"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2409.12889","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12889","pdf_url":"https://arxiv.org/pdf/2409.12889","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.12889","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100338438","display_name":"Peng Chen","orcid":"https://orcid.org/0000-0001-8582-7661","observed_orcids":["https://orcid.org/0000-0001-8582-7661"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Peng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114402301","display_name":"Pi Bu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bu, Pi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Song, Jun","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Gao, Yuan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5073856221","display_name":"Bo Zheng","orcid":"https://orcid.org/0000-0002-4037-6315","observed_orcids":["https://orcid.org/0000-0002-4037-6315"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.614300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.614300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.295199990272522,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.033799998462200165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.9549999833106995},{"id":"https://openalex.org/keywords/action-role-playing-games","display_name":"action role-playing games","score":0.9210000038146973},{"id":"https://openalex.org/keywords/video-game-agents","display_name":"video game agents","score":0.6539999842643738},{"id":"https://openalex.org/keywords/black-myth-wukong","display_name":"Black Myth: Wukong","score":0.949999988079071},{"id":"https://openalex.org/keywords/combat-tasks","display_name":"combat tasks","score":0.5239999890327454},{"id":"https://openalex.org/keywords/action-planning","display_name":"action planning","score":0.7590000033378601},{"id":"https://openalex.org/keywords/multimodal-agents","display_name":"multimodal agents","score":0.7289999723434448}],"concepts":[{"id":"https://openalex.org/C519517224","wikidata":"https://www.wikidata.org/wiki/Q12827256","display_name":"Mythology","level":2,"score":0.7521880865097046},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.7294612526893616},{"id":"https://openalex.org/C112698675","wikidata":"https://www.wikidata.org/wiki/Q37038","display_name":"Advertising","level":1,"score":0.3494533896446228},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.19113904237747192},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.18260520696640015},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.14694565534591675},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.07624426484107971},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2409.12889","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12889","pdf_url":"https://arxiv.org/pdf/2409.12889","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2409.12889","doi":"https://doi.org/10.48550/arxiv.2409.12889","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2409.12889","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.12889","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.12889","pdf_url":"https://arxiv.org/pdf/2409.12889","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415062357","https://openalex.org/W6947894701","https://openalex.org/W7163986129","https://openalex.org/W4404649840","https://openalex.org/W7138403479","https://openalex.org/W7139045847","https://openalex.org/W4417292648","https://openalex.org/W4415103088","https://openalex.org/W4412888052","https://openalex.org/W6967086278"],"abstract_inverted_index":{"Recently,":[0],"large":[1],"language":[2,65],"model":[3],"(LLM)-based":[4],"agents":[5,23,68,242],"have":[6,30,70],"made":[7,254],"significant":[8],"advances":[9],"across":[10],"various":[11],"fields.":[12],"One":[13],"of":[14,50,63,137,201,224],"the":[15,48,61,134,154,214],"most":[16],"popular":[17],"research":[18,130,231],"areas":[19],"involves":[20],"applying":[21,240],"these":[22,28,86,119],"to":[24,35,77,132,216,233],"video":[25],"games.":[26,59],"Traditionally,":[27],"methods":[29,105],"relied":[31],"on":[32,159],"game":[33,246],"APIs":[34,51],"access":[36],"in-game":[37],"environmental":[38],"and":[39,52,113,145,161,181,186,206,220,226,237,250],"action":[40,98,147,203,245],"data.":[41],"However,":[42],"this":[43,167],"approach":[44],"is":[45],"limited":[46],"by":[47],"availability":[49],"does":[53],"not":[54],"reflect":[55],"how":[56],"humans":[57],"play":[58],"With":[60],"advent":[62],"vision":[64],"models":[66],"(VLMs),":[67],"now":[69],"enhanced":[71],"visual":[72,83,208],"understanding":[73],"capabilities,":[74],"enabling":[75],"them":[76],"interact":[78],"with":[79,156],"games":[80,100],"using":[81],"only":[82],"inputs.":[84],"Despite":[85],"advances,":[87],"current":[88],"approaches":[89],"still":[90],"face":[91],"challenges":[92],"in":[93,97,140,222,243],"action-oriented":[94],"tasks,":[95],"particularly":[96],"role-playing":[99],"(ARPGs),":[101],"where":[102],"reinforcement":[103],"learning":[104],"are":[106],"prevalent":[107],"but":[108],"suffer":[109],"from":[110],"poor":[111],"generalization":[112],"require":[114],"extensive":[115],"training.":[116],"To":[117],"address":[118],"limitations,":[120],"we":[121,170,190],"select":[122],"an":[123,202],"ARPG,":[124],"``Black":[125],"Myth:":[126],"Wukong'',":[127],"as":[128],"a":[129,173,192,207],"platform":[131],"explore":[133],"capability":[135],"boundaries":[136],"existing":[138],"VLMs":[139,165],"scenarios":[141],"requiring":[142],"visual-only":[143],"input":[144],"complex":[146,244],"output.":[148],"We":[149],"define":[150],"12":[151],"tasks":[152,219],"within":[153],"game,":[155],"75%":[157],"focusing":[158],"combat,":[160],"incorporate":[162],"several":[163],"state-of-the-art":[164],"into":[166],"benchmark.":[168],"Additionally,":[169],"will":[171,252],"release":[172],"human":[174],"operation":[175,182],"dataset":[176],"containing":[177],"recorded":[178],"gameplay":[179],"videos":[180],"logs,":[183],"including":[184],"mouse":[185],"keyboard":[187],"actions.":[188],"Moreover,":[189],"propose":[191],"novel":[193],"VARP":[194],"(Vision":[195],"Action":[196],"Role-Playing)":[197],"agent":[198],"framework,":[199],"consisting":[200],"planning":[204],"system":[205],"trajectory":[209],"system.":[210],"Our":[211],"framework":[212],"demonstrates":[213],"ability":[215],"perform":[217],"basic":[218],"succeed":[221],"90%":[223],"easy":[225],"medium-level":[227],"combat":[228],"scenarios.":[229],"This":[230],"aims":[232],"provide":[234],"new":[235],"insights":[236],"directions":[238],"for":[239],"multimodal":[241],"environments.":[247],"The":[248],"code":[249],"datasets":[251],"be":[253],"available":[255],"at":[256],"https://varp-agent.github.io/.":[257]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-09-27T08:07:27.782135","created_date":"2024-10-25T00:00:00"}