{"id":"https://openalex.org/W7154731960","doi":"https://doi.org/10.48550/arxiv.2604.15280","title":"Why Do Vision Language Models Struggle To Recognize Human Emotions?","display_name":"Why Do Vision Language Models Struggle To Recognize Human Emotions?","publication_year":2026,"publication_date":"2026-04-16","ids":{"openalex":"https://openalex.org/W7154731960","doi":"https://doi.org/10.48550/arxiv.2604.15280"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.15280","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.15280","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.15280","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133898442","display_name":"Madhav Agarwal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agarwal, Madhav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054437456","display_name":"Sotirios A. Tsaftaris","orcid":"https://orcid.org/0000-0002-8795-9294"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tsaftaris, Sotirios A.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133843179","display_name":"Laura Sevilla-Lara","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sevilla-Lara, Laura","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133896013","display_name":"Steven McDonagh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"McDonagh, Steven","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11094","display_name":"Face Recognition and Perception","score":0.40860000252723694,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11094","display_name":"Face Recognition and Perception","score":0.40860000252723694,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.3564000129699707,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.07169999927282333,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.661300003528595},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6111999750137329},{"id":"https://openalex.org/keywords/frame","display_name":"Frame (networking)","score":0.5221999883651733},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.4997999966144562},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.4875999987125397},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.423799991607666},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.3824999928474426},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.38190001249313354}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.661300003528595},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6111999750137329},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6015999913215637},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.5221999883651733},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4997999966144562},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.4875999987125397},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.427700012922287},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.423799991607666},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4025999903678894},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3926999866962433},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.3824999928474426},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.38190001249313354},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.3538999855518341},{"id":"https://openalex.org/C90559484","wikidata":"https://www.wikidata.org/wiki/Q778379","display_name":"Expression (computer science)","level":2,"score":0.33629998564720154},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.33559998869895935},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.3073999881744385},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3012000024318695},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C2993724205","wikidata":"https://www.wikidata.org/wiki/Q315","display_name":"Human language","level":2,"score":0.2770000100135803},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.273499995470047},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.27250000834465027},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.25699999928474426},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.2529999911785126},{"id":"https://openalex.org/C104114177","wikidata":"https://www.wikidata.org/wiki/Q79782","display_name":"Motion (physics)","level":2,"score":0.25220000743865967}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.15280","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.15280","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.15280","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.15280","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Understanding":[0],"emotions":[1,55,122],"is":[2,42,139,192,241],"a":[3,34,175,212,217],"fundamental":[4],"ability":[5],"for":[6,28,37,141,178],"intelligent":[7],"systems":[8],"to":[9,12,52,57,74,108,117,148,245],"be":[10],"able":[11],"interact":[13],"with":[14,195],"humans.":[15],"Vision-language":[16],"models":[17],"(VLMs)":[18],"have":[19],"made":[20],"tremendous":[21],"progress":[22],"in":[23,171,190],"the":[24,46,68,81,104,164,184,196,206,224,246,260],"last":[25],"few":[26],"years":[27],"many":[29],"visual":[30,256],"tasks,":[31],"potentially":[32],"offering":[33],"promising":[35],"solution":[36],"understanding":[38,142],"emotions.":[39,143],"However,":[40,144],"it":[41],"surprising":[43],"that":[44,80,131,168,183,222],"even":[45,59],"most":[47,207],"sophisticated":[48],"contemporary":[49],"VLMs":[50,72,110,145,191],"struggle":[51,73],"recognize":[53,75],"human":[54,76],"or":[56],"outperform":[58],"specialized":[60],"vision-only":[61],"classifiers.":[62],"In":[63],"this":[64,112],"paper":[65],"we":[66,215],"ask":[67],"question":[69],"\"Why":[70],"do":[71],"emotions?\",":[77],"and":[78,84,103,163],"observe":[79],"inherently":[82,193],"continuous":[83],"dynamic":[85],"task":[86],"of":[87,166,199],"facial":[88],"expression":[89],"recognition":[90],"(DFER)":[91],"exposes":[92],"two":[93],"critical":[94,140,208],"VLM":[95,247],"vulnerabilities.":[96],"First,":[97],"emotion":[98,179],"datasets":[99],"are":[100,146,158,204],"naturally":[101],"long-tailed,":[102],"web-scale":[105],"data":[106,257],"used":[107,189],"pre-train":[109],"exacerbates":[111],"head-class":[113],"bias,":[114],"causing":[115],"them":[116,232],"systematically":[118],"collapse":[119],"rare,":[120],"under-represented":[121],"into":[123,233],"common":[124,134],"categories.":[125],"We":[126,181],"propose":[127,216],"alternative":[128],"sampling":[129,187],"strategies":[130],"prevent":[132],"favoring":[133],"concepts.":[135],"Second,":[136],"temporal":[137,150,186],"information":[138,151,225],"unable":[147],"represent":[149],"over":[152],"dense":[153],"frame":[154],"sequences,":[155],"as":[156,243],"they":[157],"limited":[159],"by":[160,229],"context":[161,219,240],"size":[162],"number":[165],"tokens":[167],"can":[169],"fit":[170],"memory,":[172],"which":[173,203],"poses":[174],"clear":[176],"challenge":[177],"recognition.":[180],"demonstrate":[182],"sparse":[185,249],"strategy":[188,221],"misaligned":[194],"fleeting":[197],"nature":[198],"micro-expressions":[200],"(0.25-0.5":[201],"seconds),":[202],"often":[205],"affective":[209],"signal.":[210],"As":[211],"diagnostic":[213],"probe,":[214],"multi-stage":[218],"enrichment":[220],"utilizes":[223],"from":[226,254],"\"in-between\"":[227],"frames":[228],"first":[230],"converting":[231],"natural":[234],"language":[235],"summaries.":[236],"This":[237],"enriched":[238],"textual":[239],"provided":[242],"input":[244],"alongside":[248],"keyframes,":[250],"preventing":[251],"attentional":[252],"dilution":[253],"excessive":[255],"while":[258],"preserving":[259],"emotional":[261],"trajectory.":[262]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-18T00:00:00"}