{"id":"https://openalex.org/W4416613140","doi":"https://doi.org/10.52202/085713-0457","title":"Spatial-MLLM: Boosting MLLM Capabilities in Visual-based Spatial Intelligence","display_name":"Spatial-MLLM: Boosting MLLM Capabilities in Visual-based Spatial Intelligence","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416613140","doi":"https://doi.org/10.52202/085713-0457"},"language":"en","primary_location":{"id":"doi:10.52202/085713-0457","doi":"https://doi.org/10.52202/085713-0457","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0457","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.23747","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104143024","display_name":"Diankun Wu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Diankun Wu","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030451822","display_name":"Fangfu Liu","orcid":"https://orcid.org/0009-0007-9422-218X","observed_orcids":["https://orcid.org/0009-0007-9422-218X"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fangfu Liu","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070505006","display_name":"Yi\u2010Hsin Hung","orcid":"https://orcid.org/0000-0002-9940-2055","observed_orcids":["https://orcid.org/0000-0002-9940-2055"]},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Yi-Hsin Hung","raw_affiliation_strings":["National Taiwan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Taiwan University","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013973037","display_name":"Yueqi Duan","orcid":"https://orcid.org/0000-0002-1190-6663","observed_orcids":["https://orcid.org/0000-0002-1190-6663"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yueqi Duan","raw_affiliation_strings":["Department of Electronic Engineering, Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":66.8363,"has_fulltext":true,"cited_by_count":187,"citation_normalized_percentile":{"value":0.99896329,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"15211","last_page":"15239"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.9559999704360962},{"id":"https://openalex.org/keywords/spatial-reasoning","display_name":"spatial reasoning","score":0.8420000076293945},{"id":"https://openalex.org/keywords/dual-encoder-architecture","display_name":"dual-encoder architecture","score":0.7080000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7997000217437744},{"id":"https://openalex.org/C155911833","wikidata":"https://www.wikidata.org/wiki/Q3817354","display_name":"Spatial intelligence","level":2,"score":0.714900016784668},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7008000016212463},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6392999887466431},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5123999714851379},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5078999996185303},{"id":"https://openalex.org/C159620131","wikidata":"https://www.wikidata.org/wiki/Q1938983","display_name":"Spatial analysis","level":2,"score":0.4921000003814697},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.4921000003814697},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.45669999718666077},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.4293999969959259},{"id":"https://openalex.org/C64754055","wikidata":"https://www.wikidata.org/wiki/Q7574053","display_name":"Spatial contextual awareness","level":2,"score":0.4278999865055084},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4223000109195709},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.41499999165534973},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.40639999508857727},{"id":"https://openalex.org/C27511587","wikidata":"https://www.wikidata.org/wiki/Q2178623","display_name":"Spatial relation","level":2,"score":0.3991999924182892},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.38769999146461487},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.351500004529953},{"id":"https://openalex.org/C2780103172","wikidata":"https://www.wikidata.org/wiki/Q1309721","display_name":"Visual Objects","level":3,"score":0.3409000039100647},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.3407999873161316},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.33070001006126404},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3280999958515167},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2773999869823456},{"id":"https://openalex.org/C2780297707","wikidata":"https://www.wikidata.org/wiki/Q4895393","display_name":"Landmark","level":2,"score":0.27379998564720154},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.25600001215934753},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.251800000667572}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-0457","doi":"https://doi.org/10.52202/085713-0457","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0457","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.23747","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2505.23747","pdf_url":"https://arxiv.org/pdf/2505.23747","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.23747","doi":"https://doi.org/10.48550/arxiv.2505.23747","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.23747","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.23747","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2505.23747","pdf_url":"https://arxiv.org/pdf/2505.23747","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4782915672","display_name":null,"funder_award_id":"62206147","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416613140.pdf","grobid_xml":"https://content.openalex.org/works/W4416613140.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W4366850747","https://openalex.org/W4378711593","https://openalex.org/W4386044453","https://openalex.org/W4386185600","https://openalex.org/W4388964408","https://openalex.org/W4389519620","https://openalex.org/W4392679398","https://openalex.org/W4392972344","https://openalex.org/W4398156429","https://openalex.org/W4399655895","https://openalex.org/W4403370767","https://openalex.org/W4403748156","https://openalex.org/W4404330874","https://openalex.org/W4404784276","https://openalex.org/W4406033487","https://openalex.org/W4415308770","https://openalex.org/W4415795316","https://openalex.org/W7108068597","https://openalex.org/W7160158679","https://openalex.org/W7160242965"],"related_works":["https://openalex.org/W7123654354","https://openalex.org/W7160905493","https://openalex.org/W7133351226","https://openalex.org/W7163709696","https://openalex.org/W4416453804","https://openalex.org/W4415320074","https://openalex.org/W7165360920","https://openalex.org/W7213048247","https://openalex.org/W4416017089","https://openalex.org/W7106862600"],"abstract_inverted_index":{"Recent":[0],"advancements":[1],"in":[2,42,218],"Multimodal":[3],"Large":[4],"Language":[5],"Models":[6],"(MLLMs)":[7],"have":[8],"significantly":[9],"enhanced":[10,144],"performance":[11,217],"on":[12,29,76,178,199,208],"2D":[13,46,68,109],"visual":[14,78,97,110,126,141],"tasks.":[15,228],"However,":[16],"improving":[17],"their":[18,40],"spatial":[19,37,64,119,145,182,224],"intelligence":[20],"remains":[21],"a":[22,59,104,107,117,150,165,189,219],"challenge.":[23],"Existing":[24],"3D":[25,31,118,130],"MLLMs":[26,73],"always":[27],"rely":[28,75],"additional":[30],"or":[32,51],"2.5D":[33],"data":[34],"to":[35,88,112],"incorporate":[36],"awareness,":[38],"restricting":[39],"utility":[41],"scenarios":[43],"with":[44],"only":[45],"inputs,":[47],"such":[48],"as":[49],"images":[50],"videos.":[52],"In":[53],"this":[54],"paper,":[55],"we":[56,102,148,187],"present":[57],"Spatial-MLLM,":[58],"novel":[60],"framework":[61],"for":[62,81,143,181],"visual-based":[63,223],"reasoning":[65,227],"from":[66,94,121,192],"purely":[67],"observations.":[69],"Unlike":[70],"conventional":[71],"video":[72,166],"which":[74,158],"CLIP-based":[77],"encoders":[79],"optimized":[80],"semantic":[82,114],"understanding,":[83],"our":[84],"key":[85],"insight":[86],"is":[87],"unleash":[89],"the":[90,95,122,125,160,175,197],"strong":[91],"structure":[92,131],"prior":[93],"feed-forward":[96],"geometry":[98,127],"foundation":[99],"model.":[100],"Specifically,":[101],"propose":[103,149],"dual-encoder":[105],"architecture:":[106],"pretrained":[108],"encoder":[111],"extract":[113,129],"features,":[115],"and":[116,195,204,226],"encoder-initialized":[120],"backbone":[123],"of":[124,164,222],"model-to":[128],"features.":[132],"A":[133],"connector":[134],"then":[135],"integrates":[136],"both":[137],"features":[138],"into":[139],"unified":[140],"tokens":[142],"understanding.":[146],"Furthermore,":[147],"space-aware":[151],"frame":[152],"sampling":[153],"strategy":[154],"at":[155],"inference":[156],"time,":[157],"selects":[159],"spatially":[161],"informative":[162],"frames":[163,179],"sequence,":[167],"ensuring":[168],"that":[169,213],"even":[170],"under":[171],"limited":[172],"token":[173],"length,":[174],"model":[176,198],"focuses":[177],"critical":[180],"reasoning.":[183],"Beyond":[184],"architecture":[185],"improvements,":[186],"construct":[188],"training":[190],"dataset":[191],"multiple":[193],"sources":[194],"train":[196],"it":[200],"using":[201],"supervised":[202],"fine-tuning":[203],"GRPO.":[205],"Extensive":[206],"experiments":[207],"various":[209],"real-world":[210],"datasets":[211],"demonstrate":[212],"Spatial-MLLM":[214],"achieves":[215],"state-of-the-art":[216],"wide":[220],"range":[221],"understanding":[225],"Project":[229],"page:":[230],"https://diankun-wu.github.io/Spatial-MLLM/.":[231]},"counts_by_year":[{"year":2026,"cited_by_count":144},{"year":2025,"cited_by_count":43}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}