{"id":"https://openalex.org/W7155491541","doi":"https://doi.org/10.48550/arxiv.2604.21911","title":"When Prompts Override Vision: Prompt-Induced Hallucinations in LVLMs","display_name":"When Prompts Override Vision: Prompt-Induced Hallucinations in LVLMs","publication_year":2026,"publication_date":"2026-04-23","ids":{"openalex":"https://openalex.org/W7155491541","doi":"https://doi.org/10.48550/arxiv.2604.21911"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.21911","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.21911","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.21911","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5099124181","display_name":"Pegah Khayatan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khayatan, Pegah","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134561601","display_name":"Jayneel Parekh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Parekh, Jayneel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037619048","display_name":"Arnaud Dapogny","orcid":"https://orcid.org/0000-0002-0074-8719"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dapogny, Arnaud","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111151249","display_name":"Mustafa Shukor","orcid":"https://orcid.org/0000-0002-3100-6689"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shukor, Mustafa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134540018","display_name":"Alasdair Newson","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Newson, Alasdair","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134533014","display_name":"Matthieu Cord","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cord, Matthieu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.31529998779296875,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.31529998779296875,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2004999965429306,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.05119999870657921,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hallucinating","display_name":"Hallucinating","score":0.8203999996185303},{"id":"https://openalex.org/keywords/visual-hallucination","display_name":"Visual Hallucination","score":0.5566999912261963},{"id":"https://openalex.org/keywords/prior-probability","display_name":"Prior probability","score":0.54339998960495},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.48399999737739563},{"id":"https://openalex.org/keywords/grounded-theory","display_name":"Grounded theory","score":0.48330000042915344},{"id":"https://openalex.org/keywords/dominance","display_name":"Dominance (genetics)","score":0.37630000710487366},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.3431999981403351}],"concepts":[{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.8203999996185303},{"id":"https://openalex.org/C2908998935","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Visual Hallucination","level":2,"score":0.5566999912261963},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.54339998960495},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.526199996471405},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.49129998683929443},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.48399999737739563},{"id":"https://openalex.org/C156325361","wikidata":"https://www.wikidata.org/wiki/Q1152864","display_name":"Grounded theory","level":3,"score":0.48330000042915344},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4602999985218048},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40310001373291016},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38190001249313354},{"id":"https://openalex.org/C151913843","wikidata":"https://www.wikidata.org/wiki/Q3454555","display_name":"Dominance (genetics)","level":3,"score":0.37630000710487366},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3431999981403351},{"id":"https://openalex.org/C3020402766","wikidata":"https://www.wikidata.org/wiki/Q104376712","display_name":"Prior information","level":2,"score":0.335999995470047},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2939000129699707},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2782999873161316},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2757999897003174},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26739999651908875},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.26669999957084656},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.25529998540878296},{"id":"https://openalex.org/C2993048729","wikidata":"https://www.wikidata.org/wiki/Q220821","display_name":"Visual methods","level":2,"score":0.2549000084400177}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.21911","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.21911","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.21911","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.21911","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Despite":[0],"impressive":[1],"progress":[2],"in":[3,22,31],"capabilities":[4],"of":[5,38,45,53],"large":[6],"vision-language":[7],"models":[8],"(LVLMs),":[9],"these":[10,54],"systems":[11],"remain":[12],"vulnerable":[13],"to":[14,33,67,72,137],"hallucinations,":[15],"i.e.,":[16],"outputs":[17],"that":[18,81,131,146],"are":[19],"not":[20],"grounded":[21,120,139],"the":[23,39,43,46,50,70,135,152],"visual":[24,167],"input.":[25],"Prior":[26],"work":[27],"has":[28],"attributed":[29],"hallucinations":[30,82,102],"LVLMs":[32,116],"factors":[34,55,75],"such":[35],"as":[36],"limitations":[37],"vision":[40],"backbone":[41],"or":[42,159],"dominance":[44],"language":[47],"component,":[48],"yet":[49],"relative":[51],"importance":[52],"remains":[56],"unclear.":[57],"To":[58,100,170],"resolve":[59],"this":[60],"ambiguity,":[61],"We":[62,144],"propose":[63,109],"HalluScope,":[64],"a":[65,111,127],"benchmark":[66],"better":[68],"understand":[69],"extent":[71],"which":[73],"different":[74],"induce":[76],"hallucinations.":[77],"Our":[78],"analysis":[79],"indicates":[80],"largely":[83],"stem":[84],"from":[85],"excessive":[86],"reliance":[87],"on":[88,162],"textual":[89,98,105],"priors":[90],"and":[91,166,173,186],"background":[92],"knowledge,":[93],"especially":[94],"information":[95],"introduced":[96],"through":[97],"instructions.":[99],"mitigate":[101],"induced":[103],"by":[104],"instruction":[106],"priors,":[107],"we":[108,132,176],"HalluVL-DPO,":[110],"framework":[112],"for":[113],"fine-tuning":[114],"off-the-shelf":[115],"towards":[117],"more":[118],"visually":[119],"responses.":[121],"HalluVL-DPO":[122],"leverages":[123],"preference":[124,183],"optimization":[125],"using":[126],"curated":[128],"training":[129,184],"dataset":[130],"construct,":[133],"guiding":[134],"model":[136,149],"prefer":[138],"responses":[140],"over":[141],"hallucinated":[142],"ones.":[143],"demonstrate":[145],"our":[147,180],"optimized":[148],"effectively":[150],"mitigates":[151],"targeted":[153],"hallucination":[154,164],"failure":[155],"mode,":[156],"while":[157],"preserving":[158],"improving":[160],"performance":[161],"other":[163],"benchmarks":[165],"capability":[168],"evaluations.":[169],"support":[171],"reproducibility":[172],"further":[174],"research,":[175],"will":[177],"publicly":[178],"release":[179],"evaluation":[181],"benchmark,":[182],"dataset,":[185],"code":[187],"at":[188],"https://pegah-kh.github.io/projects/prompts-override-vision/":[189],".":[190]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-04-25T00:00:00"}