{"id":"https://openalex.org/W4404783798","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.62","title":"RULE: Reliable Multimodal RAG for Factuality in Medical Vision Language Models","display_name":"RULE: Reliable Multimodal RAG for Factuality in Medical Vision Language Models","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4404783798","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.62"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.emnlp-main.62","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.emnlp-main.62","pdf_url":"https://aclanthology.org/2024.emnlp-main.62.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2024.emnlp-main.62.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101776868","display_name":"Peng Xia","orcid":"https://orcid.org/0000-0003-3113-7408","observed_orcids":["https://orcid.org/0000-0003-3113-7408"]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peng Xia","raw_affiliation_strings":["UNC-Chapel Hill ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC-Chapel Hill ,","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114172387","display_name":"Kangyu Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kangyu Zhu","raw_affiliation_strings":["Brown University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University ,","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101763206","display_name":"Haoran Li","orcid":"https://orcid.org/0000-0002-8435-2489","observed_orcids":["https://orcid.org/0000-0002-8435-2489"]},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Haoran Li","raw_affiliation_strings":["PolyU ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"PolyU ,","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077961759","display_name":"Hongtu Zhu B","orcid":"https://orcid.org/0000-0002-6781-2690","observed_orcids":["https://orcid.org/0000-0002-6781-2690"]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hongtu Zhu","raw_affiliation_strings":["UNC-Chapel Hill ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC-Chapel Hill ,","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043039413","display_name":"Yun Li","orcid":"https://orcid.org/0000-0002-5913-4551","observed_orcids":["https://orcid.org/0000-0002-5913-4551"]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yun Li","raw_affiliation_strings":["UNC-Chapel Hill ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC-Chapel Hill ,","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100438653","display_name":"Gang Li","orcid":"https://orcid.org/0000-0001-9585-1382","observed_orcids":["https://orcid.org/0000-0001-9585-1382"]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gang Li","raw_affiliation_strings":["UNC-Chapel Hill ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC-Chapel Hill ,","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115593378","display_name":"Linjun Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Linjun Zhang","raw_affiliation_strings":["Rutgers University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051534896","display_name":"Huaxiu Yao","orcid":"https://orcid.org/0000-0002-8691-9629","observed_orcids":["https://orcid.org/0000-0002-8691-9629"]},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huaxiu Yao","raw_affiliation_strings":["UNC-Chapel Hill ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNC-Chapel Hill ,","institution_ids":["https://openalex.org/I114027177"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":13.3012,"has_fulltext":true,"cited_by_count":72,"citation_normalized_percentile":{"value":0.99142549,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1081","last_page":"1093"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.0034000000450760126,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9670000076293945},{"id":"https://openalex.org/keywords/factuality","display_name":"factuality","score":0.7739999890327454},{"id":"https://openalex.org/keywords/factual-accuracy","display_name":"factual accuracy","score":0.578000009059906},{"id":"https://openalex.org/keywords/preference-dataset","display_name":"preference dataset","score":0.8610000014305115},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.7049999833106995},{"id":"https://openalex.org/keywords/medical-report-generation","display_name":"medical report generation","score":0.8130000233650208},{"id":"https://openalex.org/keywords/multimodal-retrieval","display_name":"multimodal retrieval","score":0.6340000033378601}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7461889982223511},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5406500697135925},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5210506319999695},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.38551291823387146}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2024.emnlp-main.62","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.emnlp-main.62","pdf_url":"https://aclanthology.org/2024.emnlp-main.62.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2407.05131","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.05131","pdf_url":"https://arxiv.org/pdf/2407.05131","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.05131","doi":"https://doi.org/10.48550/arxiv.2407.05131","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.05131","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.emnlp-main.62","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.emnlp-main.62","pdf_url":"https://aclanthology.org/2024.emnlp-main.62.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320307791","display_name":"Cisco Systems","ror":"https://ror.org/03yt1ez60"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404783798.pdf","grobid_xml":"https://content.openalex.org/works/W4404783798.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W2925863688","https://openalex.org/W3168867926","https://openalex.org/W4327810158","https://openalex.org/W4366330503","https://openalex.org/W4366850747","https://openalex.org/W4377121462","https://openalex.org/W4386555477","https://openalex.org/W4389072473","https://openalex.org/W4389984066","https://openalex.org/W4391835988","https://openalex.org/W4392011670","https://openalex.org/W4394659038","https://openalex.org/W4402727764","https://openalex.org/W4402727851","https://openalex.org/W4402753774","https://openalex.org/W4402753927","https://openalex.org/W4402951547","https://openalex.org/W4402951755","https://openalex.org/W4403069769","https://openalex.org/W4404725396","https://openalex.org/W4404782898","https://openalex.org/W4410803261","https://openalex.org/W4411177432","https://openalex.org/W4413447271","https://openalex.org/W4415796022","https://openalex.org/W4415800508","https://openalex.org/W7103755402","https://openalex.org/W7159847229"],"related_works":["https://openalex.org/W4403579125","https://openalex.org/W4415537815","https://openalex.org/W7126073420","https://openalex.org/W4414381644","https://openalex.org/W4417077865","https://openalex.org/W4414490788","https://openalex.org/W7155058517","https://openalex.org/W7161254881","https://openalex.org/W7168035881","https://openalex.org/W4416139362"],"abstract_inverted_index":{"The":[0],"recent":[1],"emergence":[2],"of":[3,42,103,120,123,160,176],"Medical":[4],"Large":[5],"Vision":[6],"Language":[7],"Models":[8],"(Med-LVLMs)":[9],"has":[10],"enhanced":[11],"medical":[12,29,163],"diagnosis.However,":[13],"current":[14],"Med-LVLMs":[15],"frequently":[16],"encounter":[17],"factual":[18,40,179],"issues,":[19,97],"often":[20],"generating":[21],"responses":[22],"that":[23],"do":[24],"not":[25,54],"align":[26],"with":[27,69],"established":[28],"facts.Retrieval-Augmented":[30],"Generation":[31],"(RAG),":[32],"which":[33,101],"utilizes":[34],"external":[35],"knowledge,":[36],"can":[37,62,83],"improve":[38],"the":[39,70,76,117,121,144,158],"accuracy":[41],"these":[43,96],"models":[44],"but":[45],"introduces":[46],"two":[47,104],"major":[48],"challenges.First,":[49],"limited":[50],"retrieved":[51,89,124,132,153],"contexts":[52,133,154],"might":[53],"cover":[55],"all":[56],"necessary":[57],"information,":[58],"while":[59],"excessive":[60],"retrieval":[61],"introduce":[63,107],"irrelevant":[64],"and":[65,152,165,185],"inaccurate":[66],"references,":[67],"interfering":[68],"model's":[71],"generation.Second,":[72],"in":[73,92,178,187],"cases":[74],"where":[75,129],"model":[77],"originally":[78],"responds":[79],"correctly,":[80],"applying":[81],"RAG":[82],"lead":[84],"to":[85,135,142],"an":[86,173],"over-reliance":[87,130],"on":[88,127,131,149,162],"contexts,":[90],"resulting":[91],"incorrect":[93],"answers.To":[94],"address":[95],"we":[98,106,137],"propose":[99],"RULE,":[100],"consists":[102],"components.First,":[105],"a":[108,139],"provably":[109],"effective":[110],"strategy":[111],"for":[112,155],"controlling":[113],"factuality":[114],"risk":[115],"through":[116],"calibrated":[118],"selection":[119],"number":[122],"contexts.Second,":[125],"based":[126],"samples":[128],"led":[134],"errors,":[136],"curate":[138],"preference":[140],"dataset":[141],"fine-tune":[143],"model,":[145],"balancing":[146],"its":[147],"dependence":[148],"inherent":[150],"knowledge":[151],"generation.We":[156],"demonstrate":[157],"effectiveness":[159],"RULE":[161],"VQA":[164],"report":[166],"generation":[167],"tasks":[168],"across":[169],"three":[170],"datasets,":[171],"achieving":[172],"average":[174],"improvement":[175],"47.4%":[177],"accuracy.We":[180],"publicly":[181],"release":[182],"our":[183],"benchmark":[184],"code":[186],"https:":[188],"//github.com/richard-peng-xia/RULE.":[189]},"counts_by_year":[{"year":2026,"cited_by_count":34},{"year":2025,"cited_by_count":34},{"year":2024,"cited_by_count":4}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2024-11-28T00:00:00"}