{"id":"https://openalex.org/W7161401622","doi":"https://doi.org/10.1145/3796715","title":"VGL-DPO: Vision-Guided Lexical Direct Preference Optimization for Mitigating Hallucination in Multimodal Large Language Models","display_name":"VGL-DPO: Vision-Guided Lexical Direct Preference Optimization for Mitigating Hallucination in Multimodal Large Language Models","publication_year":2026,"publication_date":"2026-05-16","ids":{"openalex":"https://openalex.org/W7161401622","doi":"https://doi.org/10.1145/3796715"},"language":"en","primary_location":{"id":"doi:10.1145/3796715","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3796715","pdf_url":null,"source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3796715","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136313727","display_name":"Siyuan Li","orcid":"https://orcid.org/0000-0002-7002-5081"},"institutions":[{"id":"https://openalex.org/I75689368","display_name":"Communication University of China","ror":"https://ror.org/04facbs33","country_code":"CN","type":"education","lineage":["https://openalex.org/I75689368"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Siyuan Li","raw_affiliation_strings":["School of Data Science and Intelligent Media, Communication University of China, Beijing,\u00a0China"],"raw_orcid":"https://orcid.org/0000-0002-7002-5081","affiliations":[{"raw_affiliation_string":"School of Data Science and Intelligent Media, Communication University of China, Beijing,\u00a0China","institution_ids":["https://openalex.org/I75689368"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074997032","display_name":"F H Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Feng Wang","raw_affiliation_strings":["Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0002-0516-1618","affiliations":[{"raw_affiliation_string":"Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008542961","display_name":"Simeng Qin","orcid":"https://orcid.org/0000-0002-3286-6824"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Simeng Qin","raw_affiliation_strings":["Northeastern University, Shenyang, China"],"raw_orcid":"https://orcid.org/0000-0002-3286-6824","affiliations":[{"raw_affiliation_string":"Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043909623","display_name":"Ranjie Duan","orcid":null},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ranjie Duan","raw_affiliation_strings":["Alibaba Group, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0002-2261-4268","affiliations":[{"raw_affiliation_string":"Alibaba Group, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136316615","display_name":"Haonan Cheng","orcid":"https://orcid.org/0000-0003-3407-4318"},"institutions":[{"id":"https://openalex.org/I75689368","display_name":"Communication University of China","ror":"https://ror.org/04facbs33","country_code":"CN","type":"education","lineage":["https://openalex.org/I75689368"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haonan Cheng","raw_affiliation_strings":["Communication University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3407-4318","affiliations":[{"raw_affiliation_string":"Communication University of China, Beijing, China","institution_ids":["https://openalex.org/I75689368"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100600271","display_name":"Long Ye","orcid":"https://orcid.org/0000-0002-3562-5612"},"institutions":[{"id":"https://openalex.org/I75689368","display_name":"Communication University of China","ror":"https://ror.org/04facbs33","country_code":"CN","type":"education","lineage":["https://openalex.org/I75689368"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Long Ye","raw_affiliation_strings":["Communication University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3562-5612","affiliations":[{"raw_affiliation_string":"Communication University of China, Beijing, China","institution_ids":["https://openalex.org/I75689368"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.60526374,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"22","issue":"7","first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.21629999577999115,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.21629999577999115,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.094200000166893,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08969999849796295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hallucinating","display_name":"Hallucinating","score":0.8478000164031982},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.7027999758720398},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6930999755859375},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.4602999985218048},{"id":"https://openalex.org/keywords/salience","display_name":"Salience (neuroscience)","score":0.39430001378059387},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3531999886035919}],"concepts":[{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.8478000164031982},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8130999803543091},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.7027999758720398},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6930999755859375},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6047000288963318},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4787999987602234},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.4602999985218048},{"id":"https://openalex.org/C108154423","wikidata":"https://www.wikidata.org/wiki/Q1469792","display_name":"Salience (neuroscience)","level":2,"score":0.39430001378059387},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3547999858856201},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3531999886035919},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.3382999897003174},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.305400013923645},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.26669999957084656},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C134537474","wikidata":"https://www.wikidata.org/wiki/Q17144832","display_name":"Naturalness","level":2,"score":0.25769999623298645}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3796715","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3796715","pdf_url":null,"source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3796715","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3796715","pdf_url":null,"source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W2962735233","https://openalex.org/W4386071687","https://openalex.org/W4389335072","https://openalex.org/W4402703048","https://openalex.org/W4402704633","https://openalex.org/W4402716047","https://openalex.org/W4402727405","https://openalex.org/W4403888461","https://openalex.org/W4405596328","https://openalex.org/W4409366758","https://openalex.org/W4415795501","https://openalex.org/W4415800635","https://openalex.org/W6950310954","https://openalex.org/W7133193597"],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"large":[1],"language":[2],"models":[3,190],"(MLLMs)":[4],"have":[5,41],"achieved":[6],"significant":[7],"advancements":[8],"in":[9,67,85,108,153,199],"multimodal":[10],"understanding,":[11],"reasoning,":[12],"and":[13,80,120,155,202],"interaction.":[14],"However,":[15,52],"they":[16],"still":[17],"suffer":[18],"from":[19,27],"hallucination,":[20],"where":[21],"the":[22,28,32,61,104,117,147,162,165,175],"generated":[23],"text":[24,69],"often":[25],"deviates":[26],"factual":[29,142,184,204],"content":[30,181],"of":[31,64,106,164],"input":[33,119],"image.":[34],"To":[35,88],"mitigate":[36],"this":[37,90],"issue,":[38],"prior":[39],"studies":[40],"primarily":[42],"employed":[43],"direct":[44],"preference":[45,50,110,157,167],"optimization":[46,134],"(DPO)":[47],"for":[48],"human":[49],"alignment.":[51],"these":[53],"approaches":[54],"treat":[55],"all":[56],"textual":[57],"words":[58,66,107,127,138,152],"equally,":[59],"neglecting":[60],"varying":[62],"significance":[63,105],"individual":[65],"grounding":[68],"generation":[70],"to":[71,116,125,141,159,178],"image":[72],"content.":[73],"This":[74,130,169],"limitation":[75],"hinders":[76],"fine-grained":[77],"semantic":[78],"alignment":[79],"consequently":[81],"constrains":[82],"their":[83,114],"effectiveness":[84],"hallucination":[86,201],"suppression.":[87],"address":[89],"limitation,":[91],"we":[92,102,145],"propose":[93],"a":[94],"vision-guided":[95],"lexical":[96],"DPO":[97],"method,":[98],"called":[99],"VGL-DPO.":[100],"Specifically,":[101],"quantify":[103],"positive":[109,154],"data":[111,158],"based":[112],"on":[113],"relevance":[115],"visual":[118],"dynamically":[121],"assign":[122],"different":[123,126],"weights":[124],"during":[128],"training.":[129],"facilitates":[131],"more":[132],"precise":[133],"by":[135],"emphasizing":[136],"critical":[137],"that":[139,192],"contribute":[140],"grounding.":[143],"Additionally,":[144],"leverage":[146],"importance":[148],"differences":[149],"between":[150],"high-significance":[151],"negative":[156,166],"adaptively":[160],"adjust":[161],"weight":[163],"loss.":[168],"dynamic":[170],"reweighting":[171],"mechanism":[172],"further":[173],"refines":[174],"model\u2019s":[176],"ability":[177],"suppress":[179],"hallucinated":[180],"while":[182],"reinforcing":[183],"accuracy.":[185,205],"Extensive":[186],"experiments":[187],"across":[188],"various":[189],"demonstrate":[191],"our":[193],"method":[194],"outperforms":[195],"existing":[196],"state-of-the-art":[197],"methods":[198],"reducing":[200],"enhancing":[203]},"counts_by_year":[],"updated_date":"2026-06-25T06:16:34.145877","created_date":"2026-05-17T00:00:00"}