{"id":"https://openalex.org/W4401117690","doi":"https://doi.org/10.1162/coli_a_00531","title":"Do Multimodal Large Language Models and Humans Ground Language Similarly?","display_name":"Do Multimodal Large Language Models and Humans Ground Language Similarly?","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4401117690","doi":"https://doi.org/10.1162/coli_a_00531"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00531","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00531","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/coli_a_00531","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101419521","display_name":"Cameron R. Jones","orcid":"https://orcid.org/0000-0002-6609-8966"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Cameron R. Jones","raw_affiliation_strings":["University of California, San Diego, Department of Cognitive Science. cameron@ucsd.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California, San Diego, Department of Cognitive Science. cameron@ucsd.edu","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043344696","display_name":"Benjamin K. Bergen","orcid":"https://orcid.org/0000-0002-9395-9151"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benjamin Bergen","raw_affiliation_strings":["University of California, San Diego, Department of Cognitive Science. bkbergen@ucsd.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California, San Diego, Department of Cognitive Science. bkbergen@ucsd.edu","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054191699","display_name":"Sean Trott","orcid":"https://orcid.org/0000-0002-6003-3731"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sean Trott","raw_affiliation_strings":["University of California, San Diego, Department of Cognitive Science. sttrott@ucsd.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California, San Diego, Department of Cognitive Science. sttrott@ucsd.edu","institution_ids":["https://openalex.org/I36258959"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5101419521"],"corresponding_institution_ids":["https://openalex.org/I36258959"],"apc_list":null,"apc_paid":null,"fwci":2.0399,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.86669672,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"50","issue":"4","first_page":"1415","last_page":"1440"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11431","display_name":"Action Observation and Synchronization","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embodied-cognition","display_name":"Embodied cognition","score":0.7454756498336792},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6638346910476685},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.6518759727478027},{"id":"https://openalex.org/keywords/meaning","display_name":"Meaning (existential)","score":0.5778071284294128},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5104936361312866},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.4976239502429962},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.49717190861701965},{"id":"https://openalex.org/keywords/sensitivity","display_name":"Sensitivity (control systems)","score":0.42425474524497986},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42239412665367126},{"id":"https://openalex.org/keywords/common-ground","display_name":"Common ground","score":0.4145314693450928},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3960312306880951},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.35878604650497437},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.32089361548423767},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2851361036300659},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.20700126886367798}],"concepts":[{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.7454756498336792},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6638346910476685},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.6518759727478027},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.5778071284294128},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5104936361312866},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.4976239502429962},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.49717190861701965},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.42425474524497986},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42239412665367126},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.4145314693450928},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3960312306880951},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.35878604650497437},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.32089361548423767},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2851361036300659},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.20700126886367798},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C24326235","wikidata":"https://www.wikidata.org/wiki/Q126095","display_name":"Electronic engineering","level":1,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/coli_a_00531","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00531","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:escholarship.org:ark:/13030/qt1s53f3qt","is_oa":true,"landing_page_url":"https://escholarship.org/uc/item/1s53f3qt","pdf_url":"https://escholarship.org/content/qt1s53f3qt/qt1s53f3qt.pdf?t=siaf3s","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"},{"id":"pmh:oai:doaj.org/article:66b3e77895b74fb0ba20378acb85dca1","is_oa":false,"landing_page_url":"https://doaj.org/article/66b3e77895b74fb0ba20378acb85dca1","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00531","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00531","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.7900000214576721,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W604097133","https://openalex.org/W1861492603","https://openalex.org/W1975909437","https://openalex.org/W2022856253","https://openalex.org/W2058419803","https://openalex.org/W2082928871","https://openalex.org/W2091301585","https://openalex.org/W2104817141","https://openalex.org/W2107019937","https://openalex.org/W2112184938","https://openalex.org/W2127716180","https://openalex.org/W2130663341","https://openalex.org/W2137010615","https://openalex.org/W2143623448","https://openalex.org/W2148848230","https://openalex.org/W2149544884","https://openalex.org/W2150375089","https://openalex.org/W2153469952","https://openalex.org/W2157853130","https://openalex.org/W2170014302","https://openalex.org/W2171622898","https://openalex.org/W2292919134","https://openalex.org/W2321200616","https://openalex.org/W2505756961","https://openalex.org/W2587573685","https://openalex.org/W2632333391","https://openalex.org/W2798881773","https://openalex.org/W2896457183","https://openalex.org/W2944339144","https://openalex.org/W2962862718","https://openalex.org/W3034723486","https://openalex.org/W3094502228","https://openalex.org/W3100307207","https://openalex.org/W3119485017","https://openalex.org/W3126792443","https://openalex.org/W3166396011","https://openalex.org/W3196974791","https://openalex.org/W3201264086","https://openalex.org/W4214829911","https://openalex.org/W4224875474","https://openalex.org/W4248040599","https://openalex.org/W4306820534","https://openalex.org/W4322718246","https://openalex.org/W4323572061","https://openalex.org/W4323653446","https://openalex.org/W4362655601","https://openalex.org/W4376117416","https://openalex.org/W4381802186","https://openalex.org/W4382202923","https://openalex.org/W4383058631","https://openalex.org/W4385245566","https://openalex.org/W4385571325","https://openalex.org/W4386071707","https://openalex.org/W4388685645","https://openalex.org/W4388691863","https://openalex.org/W4388963454","https://openalex.org/W4389523650","https://openalex.org/W4391094120","https://openalex.org/W4391215636","https://openalex.org/W4401042840","https://openalex.org/W4402754023","https://openalex.org/W6739715551","https://openalex.org/W6739901393","https://openalex.org/W6755207826","https://openalex.org/W6784333009","https://openalex.org/W6789753369","https://openalex.org/W6791353385","https://openalex.org/W6846007759","https://openalex.org/W6850015000","https://openalex.org/W6850503672","https://openalex.org/W6851781724","https://openalex.org/W6860561321","https://openalex.org/W6912494966"],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W203959209","https://openalex.org/W2110287964","https://openalex.org/W2167701463","https://openalex.org/W4307407935"],"abstract_inverted_index":{"Abstract":[0],"Large":[1,25],"Language":[2,26],"Models":[3,27],"(LLMs)":[4],"have":[5],"been":[6,81],"criticized":[7],"for":[8,187],"failing":[9,17],"to":[10,14,18,33,54,72,111,118,124,144,164,179,221,251],"connect":[11],"linguistic":[12,38,84],"meaning":[13,85],"the":[15,20,63,69,91,191,200,246,262],"world\u2014for":[16],"solve":[19],"\u201csymbol":[21],"grounding":[22,74],"problem.\u201d":[23],"Multimodal":[24],"(MLLMs)":[28],"offer":[29],"a":[30,212,228],"potential":[31],"solution":[32],"this":[34],"challenge":[35],"by":[36],"combining":[37],"representations":[39,97],"and":[40,53,95,148,155,208,243,257],"processing":[41],"with":[42,254],"other":[43,255],"modalities.":[44],"However,":[45],"much":[46],"is":[47,86,215],"still":[48],"unknown":[49],"about":[50],"exactly":[51],"how":[52],"what":[55],"degree":[56],"MLLMs":[57,121,183,250],"integrate":[58,252],"their":[59],"distinct":[60],"modalities\u2014and":[61],"whether":[62,120],"way":[64],"they":[65],"do":[66],"so":[67],"mirrors":[68],"mechanisms":[70,264],"believed":[71],"underpin":[73],"in":[75,115,133,195,245],"humans.":[76],"In":[77,138,157,171],"humans,":[78],"it":[79],"has":[80],"hypothesized":[82],"that":[83,127,176,210],"grounded":[87],"through":[88],"\u201cembodied":[89],"simulation,\u201d":[90],"activation":[92],"of":[93,135,169,204,218,235,248],"sensorimotor":[94,125,181,223],"affective":[96],"reflecting":[98],"described":[99],"experiences.":[100],"Across":[101],"four":[102],"pre-registered":[103],"studies,":[104],"we":[105,141,160,174,198],"adapt":[106],"experimental":[107],"techniques":[108],"originally":[109],"developed":[110],"investigate":[112],"embodied":[113],"simulation":[114],"human":[116,188,219,266],"comprehenders":[117],"ask":[119],"are":[122,128],"sensitive":[123],"features":[126,146],"implied":[129],"but":[130,150],"not":[131,151],"explicit":[132],"descriptions":[134],"an":[136,166],"event.":[137],"Experiment":[139,158,172,196],"1,":[140],"find":[142,175,209],"sensitivity":[143,178],"some":[145],"(color":[147],"shape)":[149],"others":[152],"(size,":[153],"orientation,":[154],"volume).":[156],"2,":[159],"identify":[161],"likely":[162,263],"bottlenecks":[163],"explain":[165],"MLLM\u2019s":[167],"lack":[168],"sensitivity.":[170],"3,":[173],"despite":[177],"implicit":[180],"features,":[182],"cannot":[184],"fully":[185],"account":[186],"behavior":[189],"on":[190,233,261],"same":[192],"task.":[193],"Finally,":[194],"4,":[197],"compare":[199],"psychometric":[201],"predictive":[202,217],"power":[203],"different":[205],"MLLM":[206],"architectures":[207],"ViLT,":[211],"single-stream":[213],"architecture,":[214],"more":[216],"responses":[220],"one":[222],"feature":[224],"(shape)":[225],"than":[226],"CLIP,":[227],"dual-encoder":[229],"architecture\u2014despite":[230],"being":[231],"trained":[232],"orders":[234],"magnitude":[236],"less":[237],"data.":[238],"These":[239],"results":[240],"reveal":[241],"strengths":[242],"limitations":[244],"ability":[247],"current":[249],"language":[253,267],"modalities,":[256],"also":[258],"shed":[259],"light":[260],"underlying":[265],"comprehension.":[268]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-27T14:10:00.468798","created_date":"2025-10-10T00:00:00"}