{"id":"https://openalex.org/W7151554428","doi":"https://doi.org/10.48550/arxiv.2604.04733","title":"Discovering Failure Modes in Vision-Language Models using RL","display_name":"Discovering Failure Modes in Vision-Language Models using RL","publication_year":2026,"publication_date":"2026-04-06","ids":{"openalex":"https://openalex.org/W7151554428","doi":"https://doi.org/10.48550/arxiv.2604.04733"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.04733","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04733","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.04733","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133070086","display_name":"Kanishk Jain","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Jain, Kanishk","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133080010","display_name":"Qian Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Qian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003441670","display_name":"Shravan Nayak","orcid":"https://orcid.org/0000-0002-5298-7121"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nayak, Shravan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087682739","display_name":"Parisa Kordjamshidi","orcid":"https://orcid.org/0000-0002-4606-1824"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kordjamshidi, Parisa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133070295","display_name":"Nishanth Anand","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anand, Nishanth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5063960231","display_name":"Aishwarya Agrawal","orcid":"https://orcid.org/0000-0002-8620-8077"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agrawal, Aishwarya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5133070086"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9490000009536743,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9490000009536743,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.007400000002235174,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.00570000009611249,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.6442000269889832},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.6018000245094299},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.4530999958515167},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.3472999930381775},{"id":"https://openalex.org/keywords/train","display_name":"Train","score":0.3407999873161316},{"id":"https://openalex.org/keywords/subject","display_name":"Subject (documents)","score":0.32019999623298645},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.3181000053882599}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7268000245094299},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.6442000269889832},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.6018000245094299},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5582000017166138},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5285000205039978},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4530999958515167},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.3472999930381775},{"id":"https://openalex.org/C190839683","wikidata":"https://www.wikidata.org/wiki/Q2448197","display_name":"Train","level":2,"score":0.3407999873161316},{"id":"https://openalex.org/C2777855551","wikidata":"https://www.wikidata.org/wiki/Q12310021","display_name":"Subject (documents)","level":2,"score":0.32019999623298645},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3181000053882599},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.3093999922275543},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.27730000019073486},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.2567000091075897},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.25110000371932983},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.04733","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04733","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.04733","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04733","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.44030407071113586,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Vision-language":[0],"Models":[1],"(VLMs),":[2],"despite":[3],"achieving":[4],"strong":[5],"performance":[6],"on":[7,99,118,134],"multimodal":[8],"benchmarks,":[9],"often":[10,37,57],"misinterpret":[11],"straightforward":[12],"visual":[13,136],"concepts":[14],"that":[15,35,113],"humans":[16],"identify":[17],"effortlessly,":[18],"such":[19,45],"as":[20,142],"counting,":[21],"spatial":[22],"reasoning,":[23],"and":[24,33,51,138],"viewpoint":[25],"understanding.":[26],"Previous":[27],"studies":[28],"manually":[29],"identified":[30],"these":[31,77],"weaknesses":[32],"found":[34],"they":[36],"stem":[38],"from":[39],"deficits":[40],"in":[41,61,67,150],"specific":[42],"skills.":[43],"However,":[44],"manual":[46],"efforts":[47],"are":[48],"costly,":[49],"unscalable,":[50],"subject":[52],"to":[53,86,123],"human":[54,105],"bias,":[55],"which":[56,151],"overlooks":[58],"subtle":[59],"details":[60,137],"favour":[62],"of":[63,71,95,159],"salient":[64],"objects,":[65],"resulting":[66],"an":[68],"incomplete":[69],"understanding":[70],"a":[72,81,100,110],"model's":[73],"vulnerabilities.":[74],"To":[75],"address":[76],"limitations,":[78],"we":[79],"propose":[80],"Reinforcement":[82],"Learning":[83],"(RL)-based":[84],"framework":[85,108,161],"automatically":[87],"discover":[88],"the":[89,119,156],"failure":[90,148],"modes":[91,149],"or":[92],"blind":[93],"spots":[94],"any":[96],"``candidate":[97],"VLM''":[98],"given":[101],"data":[102],"distribution":[103],"without":[104],"intervention.":[106],"Our":[107,127],"trains":[109],"questioner":[111],"agent":[112],"adaptively":[114],"generates":[115],"queries":[116],"based":[117],"candidate":[120],"VLM's":[121],"responses":[122],"elicit":[124],"incorrect":[125],"answers.":[126],"approach":[128],"increases":[129],"question":[130],"complexity":[131],"by":[132,162],"focusing":[133],"fine-grained":[135],"distinct":[139],"skill":[140],"compositions":[141],"training":[143],"progresses,":[144],"consequently":[145],"identifying":[146],"novel":[147],"VLMs":[152],"struggle.":[153],"We":[154],"demonstrate":[155],"broad":[157],"applicability":[158],"our":[160],"showcasing":[163],"its":[164],"generalizability":[165],"across":[166],"various":[167],"model":[168],"combinations.":[169]},"counts_by_year":[],"updated_date":"2026-04-29T06:10:49.150238","created_date":"2026-04-08T00:00:00"}