{"id":"https://openalex.org/W7160245013","doi":"https://doi.org/10.48550/arxiv.2605.01766","title":"Mitigating Multimodal LLMs Hallucinations via Relevance Propagation at Inference Time","display_name":"Mitigating Multimodal LLMs Hallucinations via Relevance Propagation at Inference Time","publication_year":2026,"publication_date":"2026-05-03","ids":{"openalex":"https://openalex.org/W7160245013","doi":"https://doi.org/10.48550/arxiv.2605.01766"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.01766","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.01766","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.01766","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120450007","display_name":"Itai Allouche","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Allouche, Itai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135336597","display_name":"Joseph Keshet","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Keshet, Joseph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2849000096321106,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2849000096321106,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.09719999879598618,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.07500000298023224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.6947000026702881},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6923999786376953},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6538000106811523},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6230999827384949},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.40849998593330383},{"id":"https://openalex.org/keywords/bayesian-inference","display_name":"Bayesian inference","score":0.39809998869895935},{"id":"https://openalex.org/keywords/prior-probability","display_name":"Prior probability","score":0.388700008392334}],"concepts":[{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.6947000026702881},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6923999786376953},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6538000106811523},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6230999827384949},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5702999830245972},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5376999974250793},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5228000283241272},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.40849998593330383},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.39809998869895935},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.388700008392334},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38350000977516174},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3765000104904175},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.33719998598098755},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32679998874664307},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C96851999","wikidata":"https://www.wikidata.org/wiki/Q848369","display_name":"Subconscious","level":3,"score":0.30820000171661377},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.2874000072479248},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.28519999980926514},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.2689000070095062},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C2777375102","wikidata":"https://www.wikidata.org/wiki/Q208351","display_name":"Disgust","level":3,"score":0.2535000145435333}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.01766","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.01766","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.01766","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.01766","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6379592418670654}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"large":[1],"language":[2,74],"models":[3,27],"(MLLMs)":[4],"have":[5],"revolutionized":[6],"the":[7,37,54,60,68,77,138],"landscape":[8],"of":[9,56,62,79],"AI,":[10],"demonstrating":[11,164],"impressive":[12],"capabilities":[13],"in":[14,48,158,167],"tackling":[15],"complex":[16],"vision":[17,160],"and":[18,118,161,169,184,188],"audio-language":[19],"tasks.":[20],"However,":[21],"a":[22,66,93,120],"critical":[23],"challenge":[24],"remains:":[25],"these":[26],"often":[28],"suffer":[29],"from":[30,36,44],"hallucinations,":[31],"generating":[32],"outputs":[33],"that":[34,123,179],"diverge":[35],"provided":[38],"perceptual":[39,63,128],"inputs.":[40,64,129],"This":[41,130],"tendency":[42],"stems":[43],"an":[45],"inherent":[46],"imbalance":[47],"modality":[49,104,182],"utilization":[50],"during":[51,106],"inference,":[52],"where":[53],"dominance":[55],"textual":[57,73],"tokens":[58],"undermines":[59],"potential":[61],"As":[65],"result,":[67],"model":[69,144],"frequently":[70],"resorts":[71],"to":[72,97,114,137],"priors":[75],"at":[76],"expense":[78],"grounded":[80],"evidence.":[81],"To":[82],"tackle":[83],"this":[84],"issue,":[85],"we":[86],"propose":[87],"Learning":[88],"Inference-time":[89],"Modality":[90],"Enhancement":[91],"(LIME),":[92],"training-free":[94],"framework":[95],"designed":[96],"bolster":[98],"multimodal":[99,156],"grounding":[100,171],"by":[101],"explicitly":[102],"enhancing":[103],"usage":[105],"decoding.":[107],"LIME":[108,153,180],"leverages":[109],"Layer-wise":[110],"Relevance":[111],"Propagation":[112],"(LRP)":[113],"quantify":[115],"token-level":[116],"contributions":[117],"defines":[119],"relevance-based":[121],"objective":[122,131],"promotes":[124],"increased":[125],"reliance":[126],"on":[127],"is":[132],"enforced":[133],"through":[134],"inference-time":[135],"updates":[136],"model's":[139],"key-value":[140],"representations,":[141],"without":[142],"modifying":[143],"parameters":[145],"or":[146],"requiring":[147],"additional":[148],"training":[149],"data.":[150],"We":[151],"evaluate":[152],"across":[154],"multiple":[155],"benchmarks":[157],"both":[159],"audio":[162],"domains,":[163],"consistent":[165],"reductions":[166],"hallucinations":[168],"enhanced":[170],"while":[172],"preserving":[173],"generation":[174],"quality.":[175],"Further":[176],"analysis":[177],"shows":[178],"increases":[181],"contribution":[183],"produces":[185],"more":[186],"localized":[187],"semantically":[189],"aligned":[190],"relevance":[191],"patterns.":[192]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-06T00:00:00"}