{"id":"https://openalex.org/W4416035722","doi":"https://doi.org/10.18653/v1/2025.emnlp-main.1255","title":"Hidden in Plain Sight: Reasoning in Underspecified and Misspecified Scenarios for Multimodal LLMs","display_name":"Hidden in Plain Sight: Reasoning in Underspecified and Misspecified Scenarios for Multimodal LLMs","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416035722","doi":"https://doi.org/10.18653/v1/2025.emnlp-main.1255"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.emnlp-main.1255","doi":"https://doi.org/10.18653/v1/2025.emnlp-main.1255","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.emnlp-main.1255","pdf_url":"https://aclanthology.org/2025.emnlp-main.1255.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2025.emnlp-main.1255.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113392281","display_name":"Qianqi Yan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qianqi Yan","raw_affiliation_strings":["University of California , Santa Barbara 2 eBay"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara 2 eBay","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100664113","display_name":"Hongquan Li","orcid":"https://orcid.org/0000-0001-6280-6227","observed_orcids":["https://orcid.org/0000-0001-6280-6227"]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hongquan Li","raw_affiliation_strings":["University of California , Santa Barbara 2 eBay"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara 2 eBay","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120308896","display_name":"Shan Jiang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shan Jiang","raw_affiliation_strings":["University of California , Santa Barbara 2 eBay"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara 2 eBay","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yang Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang Zhao","raw_affiliation_strings":["University of California , Santa Barbara 2 eBay"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara 2 eBay","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078314972","display_name":"Xinze Guan","orcid":"https://orcid.org/0009-0000-5021-1277","observed_orcids":["https://orcid.org/0009-0000-5021-1277"]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xinze Guan","raw_affiliation_strings":["University of California , Santa Barbara 2 eBay"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara 2 eBay","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006379171","display_name":"Ching-Chen Kuo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ching-Chen Kuo","raw_affiliation_strings":["University of California , Santa Barbara 2 eBay"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara 2 eBay","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100328100","display_name":"Xin Eric Wang","orcid":"https://orcid.org/0000-0003-2605-5504","observed_orcids":["https://orcid.org/0000-0003-2605-5504"]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xin Eric Wang","raw_affiliation_strings":["University of California , Santa Barbara 2 eBay"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara 2 eBay","institution_ids":["https://openalex.org/I154570441"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I154570441"],"apc_list":null,"apc_paid":null,"fwci":0.3574,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.70047587,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"24726","last_page":"24746"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.9959999918937683},{"id":"https://openalex.org/keywords/diagnostic-assessment","display_name":"diagnostic assessment","score":0.3330000042915344},{"id":"https://openalex.org/keywords/clarifying-questions","display_name":"clarification questions","score":0.7860000133514404},{"id":"https://openalex.org/keywords/user-compliance","display_name":"user compliance","score":0.6129999756813049},{"id":"https://openalex.org/keywords/trustworthy-ai","display_name":"trustworthy AI","score":0.6169999837875366},{"id":"https://openalex.org/keywords/ambiguous-instructions","display_name":"ambiguous instructions","score":0.3089999854564667},{"id":"https://openalex.org/keywords/missing-objects","display_name":"missing objects","score":0.3100000023841858}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6097000241279602},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4596000015735626},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3610000014305115},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34769999980926514},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2671000063419342},{"id":"https://openalex.org/C2780339515","wikidata":"https://www.wikidata.org/wiki/Q3074698","display_name":"Arrow","level":2,"score":0.25110000371932983},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.24699999392032623},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.2433999925851822},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.24250000715255737},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.23800000548362732}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.emnlp-main.1255","doi":"https://doi.org/10.18653/v1/2025.emnlp-main.1255","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.emnlp-main.1255","pdf_url":"https://aclanthology.org/2025.emnlp-main.1255.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.emnlp-main.1255","doi":"https://doi.org/10.18653/v1/2025.emnlp-main.1255","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.emnlp-main.1255","pdf_url":"https://aclanthology.org/2025.emnlp-main.1255.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416035722.pdf","grobid_xml":"https://content.openalex.org/works/W4416035722.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2789566302","https://openalex.org/W3034188538","https://openalex.org/W4306291189","https://openalex.org/W4322718191","https://openalex.org/W4388327684","https://openalex.org/W4389518939","https://openalex.org/W4389519254","https://openalex.org/W4389523832","https://openalex.org/W4389524320","https://openalex.org/W4390041933","https://openalex.org/W4391632129","https://openalex.org/W4395474395","https://openalex.org/W4403808928","https://openalex.org/W4404330874","https://openalex.org/W4405766390","https://openalex.org/W4412888075","https://openalex.org/W4414281281","https://openalex.org/W4414939355","https://openalex.org/W4416035797","https://openalex.org/W7203827598"],"related_works":["https://openalex.org/W4404395508","https://openalex.org/W7204196920","https://openalex.org/W4387323789","https://openalex.org/W4417132923","https://openalex.org/W7163377995","https://openalex.org/W4413144323","https://openalex.org/W4417287666","https://openalex.org/W7204906318","https://openalex.org/W6903734736","https://openalex.org/W7084119645"],"abstract_inverted_index":{"Multimodal":[0],"large":[1],"language":[2],"models":[3,113],"(MLLMs)":[4],"are":[5,14,138],"increasingly":[6],"deployed":[7],"in":[8,141,158,179,192],"open-ended,":[9],"realworld":[10],"environments":[11],"where":[12,80],"inputs":[13],"messy,":[15],"underspecified,":[16],"and":[17,76,108,110,127,176,182],"not":[18,47],"always":[19],"trustworthy.Unlike":[20],"curated":[21,92],"benchmarks,":[22],"these":[23,188],"settings":[24],"frequently":[25,139],"involve":[26],"instructions":[27],"that":[28,112,132,148],"reference":[29],"missing":[30],"objects":[31],"or":[32,39],"contradictory":[33],"facts,":[34],"rely":[35],"on":[36,49,53],"ambiguous":[37],"cues,":[38],"request":[40],"infeasible":[41],"actions.In":[42],"such":[43,74,152],"cases,":[44],"success":[45],"hinges":[46],"merely":[48],"task":[50],"execution,":[51],"but":[52,137],"the":[54,124,133],"model's":[55],"ability":[56],"to":[57,116],"detect":[58],"when":[59,121],"something":[60],"is":[61],"silently":[62],"wrong.This":[63],"paper":[64],"presents":[65],"a":[66,91,161,170],"systematic":[67],"analysis":[68],"of":[69,98,143],"how":[70],"current":[71,180],"MLLMs":[72],"handle":[73],"underspecified":[75],"misspecified":[77],"scenarios:":[78],"cases":[79],"flaws":[81],"must":[82],"be":[83],"inferred":[84],"from":[85],"context":[86],"rather":[87],"than":[88],"explicitly":[89],"stated.Using":[90],"diagnostic":[93],"suite":[94],"spanning":[95],"four":[96],"categories":[97],"real-world":[99],"failure":[100],"modes,":[101],"we":[102],"evaluate":[103],"nine":[104],"MLLMs,":[105,181],"including":[106],"o3":[107],"GPT-4o,":[109],"find":[111],"often":[114],"fail":[115],"surface":[117],"hidden":[118],"issues,":[119],"even":[120],"they":[122],"possess":[123],"necessary":[125],"perceptual":[126],"reasoning":[128,174],"skills.Explicit":[129],"prompting":[130,156],"reveals":[131],"underlying":[134],"capabilities":[135],"exist":[136],"suppressed":[140],"favor":[142],"user":[144],"compliance.We":[145],"further":[146],"show":[147],"simple":[149],"inference-time":[150],"interventions,":[151],"as":[153],"cautious":[154],"persona":[155],"and,":[157],"particular,":[159],"requiring":[160],"clarifying":[162],"question,":[163],"can":[164],"substantially":[165],"recover":[166],"performance.Our":[167],"findings":[168],"highlight":[169],"persistent":[171],"gap":[172],"between":[173],"competence":[175],"behavioral":[177],"compliance":[178],"suggest":[183],"practical":[184],"strategies":[185],"for":[186],"making":[187],"systems":[189],"more":[190],"trustworthy":[191],"underconstrained":[193],"environments.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-11-08T00:00:00"}