{"id":"https://openalex.org/W7224087975","doi":"https://doi.org/10.48550/arxiv.2610.36925","title":"Vision-Language Agents Can Learn in Their Own Latent Worlds","display_name":"Vision-Language Agents Can Learn in Their Own Latent Worlds","publication_year":2026,"publication_date":"2026-10-27","ids":{"openalex":"https://openalex.org/W7224087975","doi":"https://doi.org/10.48550/arxiv.2610.36925"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.36925","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.36925","pdf_url":"https://arxiv.org/pdf/2610.36925","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.36925","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5160008233","display_name":"Haoxi Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Li, Haoxi","raw_affiliation_strings":["Department of Computer Science and Engineering , HKUST , Hong Kong , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , HKUST , Hong Kong , China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011529251","display_name":"Sikai Bai","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Bai, Sikai","raw_affiliation_strings":["Department of Computer Science and Engineering , HKUST , Hong Kong , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , HKUST , Hong Kong , China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160047825","display_name":"Jianfei Ma","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Ma, Jianfei","raw_affiliation_strings":["School of Computing , National University of Singapore , Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing , National University of Singapore , Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135484157","display_name":"Qinglin Hou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hou, Qinglin","raw_affiliation_strings":["Department of Computer Science , University of Southern California , Los Angeles , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , University of Southern California , Los Angeles , USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160041462","display_name":"Xiaocheng Lu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Lu, Xiaocheng","raw_affiliation_strings":["Department of Computer Science and Engineering , HKUST , Hong Kong , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , HKUST , Hong Kong , China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159988691","display_name":"Tao Han","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Han, Tao","raw_affiliation_strings":["Department of Computer Science and Engineering , HKUST , Hong Kong , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , HKUST , Hong Kong , China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159993998","display_name":"Jie Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhang, Jie","raw_affiliation_strings":["Department of Computer Science and Engineering , HKUST , Hong Kong , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , HKUST , Hong Kong , China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5160013400","display_name":"Song Guo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Guo, Song","raw_affiliation_strings":["Department of Computer Science and Engineering , HKUST , Hong Kong , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , HKUST , Hong Kong , China","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.01510000042617321,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.8859999775886536},{"id":"https://openalex.org/keywords/world-models","display_name":"world models","score":0.7630000114440918},{"id":"https://openalex.org/keywords/model-based-reinforcement-learning","display_name":"model-based reinforcement learning","score":0.8349999785423279},{"id":"https://openalex.org/keywords/policy-improvement","display_name":"policy improvement","score":0.3070000112056732},{"id":"https://openalex.org/keywords/online-interaction","display_name":"online interaction","score":0.46799999475479126},{"id":"https://openalex.org/keywords/visual-environment","display_name":"visual environment","score":0.32600000500679016}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7361999750137329},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6398000121116638},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.60589998960495},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5957000255584717},{"id":"https://openalex.org/C106306483","wikidata":"https://www.wikidata.org/wiki/Q183984","display_name":"Futures contract","level":2,"score":0.45590001344680786},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4537000060081482},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.42489999532699585},{"id":"https://openalex.org/C2780980858","wikidata":"https://www.wikidata.org/wiki/Q110022","display_name":"Dual (grammatical number)","level":2,"score":0.42289999127388},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.3874000012874603},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3368000090122223},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.288100004196167},{"id":"https://openalex.org/C74072328","wikidata":"https://www.wikidata.org/wiki/Q1142726","display_name":"Intelligent agent","level":2,"score":0.27619999647140503},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C13687954","wikidata":"https://www.wikidata.org/wiki/Q4826847","display_name":"Autonomous agent","level":2,"score":0.2635999917984009}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.36925","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.36925","pdf_url":"https://arxiv.org/pdf/2610.36925","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.36925","doi":"https://doi.org/10.48550/arxiv.2610.36925","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.36925","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.36925","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.36925","pdf_url":"https://arxiv.org/pdf/2610.36925","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7224087975.pdf","grobid_xml":"https://content.openalex.org/works/W7224087975.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W6979297265","https://openalex.org/W4415247640","https://openalex.org/W4391591240","https://openalex.org/W4402365084","https://openalex.org/W7213100115","https://openalex.org/W7223308246","https://openalex.org/W4415897910","https://openalex.org/W7221816110","https://openalex.org/W7161000132","https://openalex.org/W2992977009"],"abstract_inverted_index":{"Learning":[0],"from":[1,124,134],"experience":[2],"is":[3,29,66,188],"essential":[4],"for":[5,161,168,184],"building":[6],"the":[7,49,54,95,106,120,148,158,174],"next":[8],"generation":[9],"of":[10],"agents":[11,170],"that":[12,82,164,181],"continually":[13,61,122],"expand":[14],"their":[15],"capabilities.":[16],"Pretrained":[17],"vision-language":[18],"models":[19,65,167],"(VLMs)":[20],"already":[21],"internalize":[22],"rich":[23],"world":[24,55,116,166],"knowledge,":[25],"yet":[26],"this":[27,40,71],"knowledge":[28],"rarely":[30],"leveraged":[31],"to":[32,57,97,127,137,178],"drive":[33],"policy":[34,46],"improvement":[35],"through":[36],"online":[37],"interaction.":[38],"Realizing":[39],"potential":[41],"efficiently":[42],"remains":[43],"challenging,":[44],"as":[45,86,114],"updates":[47],"shift":[48],"agent's":[50],"interaction":[51],"distribution,":[52],"requiring":[53],"model":[56,117],"adapt":[58],"accordingly,":[59],"while":[60,155],"updating":[62],"large":[63],"pretrained":[64],"computationally":[67],"expensive.":[68],"To":[69],"address":[70],"challenge,":[72],"we":[73],"introduce":[74],"Dwell,":[75],"a":[76,84,87],"model-based":[77],"reinforcement":[78],"learning":[79],"(RL)":[80],"framework":[81],"reframes":[83],"VLM":[85,169],"recurrent":[88],"latent":[89,103,135],"transition":[90],"operator,":[91],"endowing":[92],"it":[93],"with":[94],"ability":[96],"predict,":[98],"adjust,":[99],"and":[100,110,118,129,133],"act":[101],"in":[102],"space.":[104],"Leveraging":[105],"VLM's":[107],"own":[108],"representations":[109],"its":[111,139],"dual":[112],"role":[113],"both":[115],"policy,":[119],"agent":[121],"learns":[123],"real":[125],"interactions":[126],"anticipate":[128],"evaluate":[130],"action":[131],"outcomes,":[132],"imagination":[136],"improve":[138],"behavior.":[140],"Across":[141],"five":[142],"interactive":[143],"visual":[144],"environments,":[145],"Dwell":[146],"achieves":[147],"strongest":[149],"overall":[150],"performance":[151],"among":[152],"trainable":[153],"methods":[154],"substantially":[156],"reducing":[157],"compute":[159],"required":[160],"imagination,":[162],"showing":[163],"effective":[165],"need":[171,177],"not":[172],"reconstruct":[173],"future;":[175],"they":[176],"predict":[179],"futures":[180],"are":[182],"useful":[183],"action.":[185],"Our":[186],"code":[187],"publicly":[189],"available":[190],"at":[191],"https://github.com/Tom111-lhx/Dwell":[192]},"counts_by_year":[],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2026-10-29T00:00:00"}