{"id":"https://openalex.org/W7143468641","doi":"https://doi.org/10.48550/arxiv.2603.26589","title":"The Limits of Learning from Pictures and Text: Vision-Language Models and Embodied Scene Understanding","display_name":"The Limits of Learning from Pictures and Text: Vision-Language Models and Embodied Scene Understanding","publication_year":2026,"publication_date":"2026-03-27","ids":{"openalex":"https://openalex.org/W7143468641","doi":"https://doi.org/10.48550/arxiv.2603.26589"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2603.26589","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.26589","pdf_url":"https://arxiv.org/pdf/2603.26589","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.26589","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111495684","display_name":"Gillian K. Rosenberg","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rosenberg, Gillian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119001097","display_name":"Skylar Stadhard","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stadhard, Skylar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130922413","display_name":"Bruce C. Hansen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hansen, Bruce C.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5030173399","display_name":"Michelle R. Greene","orcid":"https://orcid.org/0000-0002-0597-4715","observed_orcids":["https://orcid.org/0000-0002-0597-4715"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Greene, Michelle R.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12694","display_name":"Categorization, perception, and language","score":0.010900000110268593,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11883","display_name":"Embodied and Extended Cognition","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.9129999876022339},{"id":"https://openalex.org/keywords/distributional-hypothesis","display_name":"distributional hypothesis","score":0.7689999938011169},{"id":"https://openalex.org/keywords/affordances","display_name":"affordances","score":0.843999981880188},{"id":"https://openalex.org/keywords/gricean-pragmatics","display_name":"Gricean pragmatics","score":0.9240000247955322},{"id":"https://openalex.org/keywords/visual-cognition","display_name":"visual cognition","score":0.6600000262260437},{"id":"https://openalex.org/keywords/future-prediction","display_name":"future prediction","score":0.4269999861717224},{"id":"https://openalex.org/keywords/emotional-responses","display_name":"emotional responses","score":0.7400000095367432}],"concepts":[{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.7748000025749207},{"id":"https://openalex.org/C194995250","wikidata":"https://www.wikidata.org/wiki/Q531136","display_name":"Affordance","level":2,"score":0.7494000196456909},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.6793000102043152},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5622000098228455},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.517300009727478},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5097000002861023},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.489300012588501},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.4860999882221222},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.4855000078678131},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4154999852180481},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3653999865055084},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.36309999227523804},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.35199999809265137},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.3508000075817108},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.3386000096797943},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.31679999828338623},{"id":"https://openalex.org/C2778662690","wikidata":"https://www.wikidata.org/wiki/Q3125339","display_name":"Spatial ability","level":3,"score":0.313400000333786},{"id":"https://openalex.org/C207551092","wikidata":"https://www.wikidata.org/wiki/Q508969","display_name":"Cognitive linguistics","level":3,"score":0.303600013256073},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.303600013256073},{"id":"https://openalex.org/C27511587","wikidata":"https://www.wikidata.org/wiki/Q2178623","display_name":"Spatial relation","level":2,"score":0.29919999837875366},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.298799991607666},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.2818000018596649},{"id":"https://openalex.org/C2982736386","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Statistical learning","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2800000011920929},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.26820001006126404},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C2780762811","wikidata":"https://www.wikidata.org/wiki/Q1784941","display_name":"Cosine similarity","level":3,"score":0.2556999921798706},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.25200000405311584},{"id":"https://openalex.org/C64357122","wikidata":"https://www.wikidata.org/wiki/Q1149766","display_name":"Causality (physics)","level":2,"score":0.25200000405311584},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2603.26589","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.26589","pdf_url":"https://arxiv.org/pdf/2603.26589","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2603.26589","doi":"https://doi.org/10.48550/arxiv.2603.26589","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.26589","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.26589","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.26589","pdf_url":"https://arxiv.org/pdf/2603.26589","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7117724348","https://openalex.org/W6922587374","https://openalex.org/W4389519828","https://openalex.org/W4410032048","https://openalex.org/W4415200456","https://openalex.org/W4402916210","https://openalex.org/W7153978741","https://openalex.org/W7164200813","https://openalex.org/W4412439081","https://openalex.org/W4414634303"],"abstract_inverted_index":{"What":[0],"information":[1],"is":[2,218],"sufficient":[3],"to":[4,65,109],"learn":[5],"the":[6,18,26,52,110,165,234],"full":[7],"richness":[8],"of":[9,21,51,67,112,197,228,236],"human":[10,70,113,229],"scene":[11,75,222],"understanding?":[12],"The":[13],"distributional":[14,53,212],"hypothesis":[15],"holds":[16],"that":[17,104,138,164,184,211,225,240],"statistical":[19],"co-occurrence":[20],"language":[22],"and":[23,86,142,172,216],"images":[24,215],"captures":[25],"conceptual":[27],"knowledge":[28,128,200],"underlying":[29],"visual":[30,230],"cognition.":[31],"Vision-language":[32],"models":[33],"(VLMs)":[34],"are":[35],"trained":[36],"on":[37,126],"massive":[38],"paired":[39],"text-image":[40],"corpora":[41],"but":[42,130],"lack":[43,92],"embodied":[44,199],"experience,":[45],"making":[46],"them":[47],"an":[48],"ideal":[49],"test":[50],"hypothesis.":[54],"We":[55],"report":[56],"two":[57],"experiments":[58],"comparing":[59],"descriptions":[60],"generated":[61],"by":[62,116,176],"18":[63],"VLMs":[64,122],"those":[66],"over":[68],"2000":[69],"observers":[71],"across":[72],"15":[73],"high-level":[74],"understanding":[76],"tasks,":[77,129],"spanning":[78],"general":[79,127],"knowledge,":[80],"affordances,":[81],"sensory":[82],"experiences,":[83],"affective":[84],"responses,":[85,114],"future":[87],"prediction.":[88],"Because":[89],"many":[90],"tasks":[91,137],"ground":[93],"truth":[94],"answers,":[95],"we":[96,153],"developed":[97],"a":[98,132],"Human-Calibrated":[99],"Cosine":[100],"Distance":[101],"(HCD)":[102],"metric":[103],"measures":[105],"VLM":[106],"output":[107],"similarity":[108],"distribution":[111],"scaled":[115],"within-human":[117],"variability.":[118],"In":[119,150],"Experiment":[120,151],"1,":[121],"approached":[123],"human-level":[124],"performance":[125],"showed":[131],"robust":[133],"deficit":[134,166],"for":[135,158,220],"affordance":[136,161,191],"resisted":[139],"prompt":[140],"engineering":[141],"did":[143],"not":[144,174],"improve":[145],"with":[146,194],"newer":[147],"model":[148],"releases.":[149],"2,":[152],"tested":[154],"six":[155],"mechanistic":[156],"hypotheses":[157],"explaining":[159],"this":[160],"gap,":[162],"finding":[163],"was":[167,173],"structural":[168],"rather":[169],"than":[170],"stylistic":[171],"resolved":[175],"providing":[177],"explicit":[178],"spatial":[179],"information.":[180],"Corpus":[181],"analyses":[182],"revealed":[183],"image":[185],"captioning":[186],"datasets":[187],"contain":[188],"sparse":[189],"agent-addressed":[190],"language,":[192],"consistent":[193],"Gricean":[195],"accounts":[196],"why":[198],"may":[201,232],"be":[202],"systematically":[203],"underrepresented":[204],"in":[205],"language.":[206],"Together,":[207],"these":[208],"findings":[209],"suggest":[210],"learning":[213],"from":[214],"text":[217],"insufficient":[219],"affordance-based":[221],"understanding,":[223],"implying":[224],"some":[226],"dimensions":[227],"cognition":[231],"require":[233],"kind":[235],"agent-centered,":[237],"three-dimensional":[238],"experience":[239],"no":[241],"photograph":[242],"or":[243],"caption":[244],"can":[245],"encode.":[246]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-03-31T00:00:00"}