{"id":"https://openalex.org/W7137802064","doi":"https://doi.org/10.1609/aaai.v40i10.37772","title":"Ground What You See: Hallucination-Resistant MLLMs via Caption Feedback, Diversity-Aware Sampling, and Conflict Regularization","display_name":"Ground What You See: Hallucination-Resistant MLLMs via Caption Feedback, Diversity-Aware Sampling, and Conflict Regularization","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7137802064","doi":"https://doi.org/10.1609/aaai.v40i10.37772"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i10.37772","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i10.37772","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v40i10.37772","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129646550","display_name":"Miao Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miao Pan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122913392","display_name":"Wangjie Gan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wangjie Gan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129665958","display_name":"Jintao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jintao Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129664295","display_name":"Wenqi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wenqi Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129741714","display_name":"Sun Bing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun Bing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129686293","display_name":"Jianwei Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jianwei Yin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129725086","display_name":"Xuhong Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuhong Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"10","first_page":"8242","last_page":"8250"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.873199999332428,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.873199999332428,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.016200000420212746,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.01209999993443489,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5217999815940857},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5149999856948853},{"id":"https://openalex.org/keywords/regularization","display_name":"Regularization (linguistics)","score":0.45719999074935913},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.4553999900817871},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.4503999948501587},{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.4478999972343445},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.4092000126838684},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.4023999869823456},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.3797999918460846}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.67330002784729},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6359000205993652},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5644000172615051},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5217999815940857},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5149999856948853},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.45719999074935913},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.4553999900817871},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4503999948501587},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.4478999972343445},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.4092000126838684},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.4023999869823456},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.3797999918460846},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.3774999976158142},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3626999855041504},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3562000095844269},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.32409998774528503},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.31839999556541443},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.299699991941452},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.2930000126361847},{"id":"https://openalex.org/C552651612","wikidata":"https://www.wikidata.org/wiki/Q7328862","display_name":"Visual thinking","level":2,"score":0.2903999984264374},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.2736999988555908},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2574999928474426},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2513999938964844},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v40i10.37772","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i10.37772","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/37772","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/37772","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i10.37772","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i10.37772","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7373835444450378}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"large":[1],"language":[2],"models":[3],"(MLLMs)":[4],"have":[5],"achieved":[6],"significant":[7],"results":[8,292],"in":[9,27,41,70],"various":[10],"tasks,":[11],"but":[12],"their":[13],"practical":[14],"application":[15],"is":[16,123,270],"still":[17],"severely":[18],"constrained":[19],"by":[20,258],"hallucination":[21,301],"issues,":[22,139],"which":[23],"are":[24],"particularly":[25],"prominent":[26],"reinforcement":[28],"learning":[29],"(RL)":[30],"optimization":[31,104,122],"processes.":[32],"This":[33],"paper":[34],"systematically":[35],"analyzes":[36],"the":[37,102,107,152,182,191,194,206,211,225,246,295,300,306],"causes":[38],"of":[39,154,193,210,309],"hallucinations":[40],"MLLM":[42],"under":[43],"RL":[44,63],"training,":[45,222],"identifying":[46],"three":[47,146],"key":[48,125,247],"factors:":[49],"(1)":[50],"The":[51,115,290],"model":[52,108],"relies":[53],"heavily":[54],"on":[55,181,190,205,228,263],"chained":[56],"visual":[57,71,80,155,171],"reasoning":[58,72],"to":[59,86,109,129,150,177,197,232,273],"guide":[60],"decision-making":[61],"during":[62,101,121],"training.":[64],"Thus,":[65],"error":[66],"and":[67,93,132,160,165,184,208,214,278,303],"irrelevant":[68],"information":[69],"can":[73],"easily":[74],"cause":[75],"hallucinations,":[76],"including":[77],"inaccurate":[78],"initial":[79,170],"descriptions":[81,172],"that":[82,127,294],"anchor":[83],"subsequent":[84],"inferences":[85],"incorrect":[87],"information,":[88],"as":[89,91,245],"well":[90],"redundant":[92],"broad":[94],"inferential":[95],"information;":[96],"(2)":[97],"Insufficient":[98],"exploration":[99,199],"diversity":[100],"policy":[103],"phase,":[105],"causing":[106],"output":[110],"overly":[111,280],"confident":[112],"results;":[113],"(3)":[114],"destructive":[116],"conflict":[117],"between":[118,235],"different":[119],"samples":[120,203,216],"a":[124,142,264,287],"factor":[126],"leads":[128],"false":[130],"associations":[131],"unstable":[133],"parameter":[134],"updates.":[135],"To":[136,168],"address":[137],"these":[138],"we":[140,157,174,201,238,254],"propose":[141],"solution":[143],"framework":[144],"comprising":[145],"core":[147],"modules.":[148],"First,":[149],"improve":[151],"accuracy":[153,308],"localization,":[156],"add":[158],"planning":[159],"caption":[161,183,187],"stages":[162],"before":[163],"thinking":[164],"answer":[166],"stages.":[167],"enhance":[169,198],"ability,":[173],"allow":[175],"LLMs":[176],"respond":[178],"based":[179,189,204,262],"solely":[180],"provide":[185],"corresponding":[186],"reward":[188,212,219],"quality":[192],"response.":[195],"Second,":[196],"capabilities,":[200],"classify":[202],"mean":[207],"variance":[209,220],"distribution":[213],"select":[215],"with":[217],"high":[218],"for":[221],"thereby":[223],"increasing":[224],"model's":[226],"focus":[227],"diverse":[229],"samples.":[230],"Finally,":[231],"mitigate":[233],"conflicts":[234],"training":[236],"samples,":[237],"identify":[239],"neural":[240],"tangent":[241],"kernel":[242],"(NTK)":[243],"similarity":[244,257,265],"factor.":[248],"Rather":[249],"than":[250],"minimizing":[251],"it":[252],"uniformly,":[253],"regulate":[255],"NTK":[256],"grouping":[259],"sample":[260],"pairs":[261,276],"threshold.":[266],"An":[267],"InfoNCE":[268],"loss":[269],"then":[271],"applied":[272],"pull":[274],"dissimilar":[275],"closer":[277],"push":[279],"similar":[281],"ones":[282],"apart,":[283],"guiding":[284],"interactions":[285],"toward":[286],"balanced":[288],"range.":[289],"experimental":[291],"demonstrate":[293],"proposed":[296],"method":[297],"significantly":[298],"reduces":[299],"rate":[302],"effectively":[304],"improves":[305],"inference":[307],"MLLMs.":[310]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-03-18T00:00:00"}