{"id":"https://openalex.org/W7138914713","doi":"https://doi.org/10.1609/aaai.v40i41.40801","title":"MMRAG-RFT: Two-stage Reinforcement Fine-tuning for Explainable Multi-modal Retrieval-augmented Generation","display_name":"MMRAG-RFT: Two-stage Reinforcement Fine-tuning for Explainable Multi-modal Retrieval-augmented Generation","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138914713","doi":"https://doi.org/10.1609/aaai.v40i41.40801"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i41.40801","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i41.40801","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40801/44762","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40801/44762","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103240423","display_name":"Shengwei Zhao","orcid":"https://orcid.org/0009-0004-3882-7959"},"institutions":[{"id":"https://openalex.org/I4210105595","display_name":"Institute of Art","ror":"https://ror.org/017fyx225","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210105595","https://openalex.org/I99542240"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","PL"],"is_corresponding":false,"raw_author_name":"Shengwei Zhao","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I4210105595","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jingwen Yao","orcid":null},"institutions":[{"id":"https://openalex.org/I4210105595","display_name":"Institute of Art","ror":"https://ror.org/017fyx225","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210105595","https://openalex.org/I99542240"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","PL"],"is_corresponding":false,"raw_author_name":"Jingwen Yao","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I4210105595","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015630107","display_name":"Sitong Wei","orcid":null},"institutions":[{"id":"https://openalex.org/I4210105595","display_name":"Institute of Art","ror":"https://ror.org/017fyx225","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210105595","https://openalex.org/I99542240"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","PL"],"is_corresponding":false,"raw_author_name":"Sitong Wei","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I4210105595","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013585366","display_name":"Linhai Xu","orcid":"https://orcid.org/0000-0001-7785-6913"},"institutions":[{"id":"https://openalex.org/I4210105595","display_name":"Institute of Art","ror":"https://ror.org/017fyx225","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210105595","https://openalex.org/I99542240"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","PL"],"is_corresponding":false,"raw_author_name":"Linhai Xu","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I4210105595","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100685987","display_name":"Yuying Liu","orcid":"https://orcid.org/0000-0002-7295-3308"},"institutions":[{"id":"https://openalex.org/I4210105595","display_name":"Institute of Art","ror":"https://ror.org/017fyx225","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210105595","https://openalex.org/I99542240"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","PL"],"is_corresponding":false,"raw_author_name":"Yuying Liu","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I4210105595","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130068403","display_name":"Dong Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210105595","display_name":"Institute of Art","ror":"https://ror.org/017fyx225","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210105595","https://openalex.org/I99542240"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","PL"],"is_corresponding":false,"raw_author_name":"Dong Zhang","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I4210105595","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhiqiang Tian","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiqiang Tian","raw_affiliation_strings":["School of Software Engineering, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software Engineering, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5130198828","display_name":"Shaoyi Du","orcid":null},"institutions":[{"id":"https://openalex.org/I4210105595","display_name":"Institute of Art","ror":"https://ror.org/017fyx225","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210105595","https://openalex.org/I99542240"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","PL"],"is_corresponding":false,"raw_author_name":"Shaoyi Du","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, National Engineering Research Center for Visual Information and Applications, and Institute of Artificial Intelligence and Robotics, Xi'an Jiaotong University","institution_ids":["https://openalex.org/I4210105595","https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"41","first_page":"34968","last_page":"34975"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6226000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6226000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.3253999948501587,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.006500000134110451,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8590999841690063},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6000000238418579},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.599399983882904},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.47940000891685486},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3813000023365021}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8590999841690063},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7483999729156494},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.654699981212616},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6000000238418579},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.599399983882904},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5145000219345093},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.47940000891685486},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3813000023365021},{"id":"https://openalex.org/C128838566","wikidata":"https://www.wikidata.org/wiki/Q275603","display_name":"Logic programming","level":2,"score":0.2535000145435333},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.25110000371932983}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v40i41.40801","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i41.40801","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40801/44762","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/40801","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/40801","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i41.40801","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i41.40801","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40801/44762","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1202231204","display_name":null,"funder_award_id":"2024PT-ZCK-66","funder_id":"https://openalex.org/F4320336350","funder_display_name":"Key Research and Development Projects of Shaanxi Province"},{"id":"https://openalex.org/G3833898155","display_name":null,"funder_award_id":"62088102","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3959510195","display_name":null,"funder_award_id":"2024CY2-GJHX-48","funder_id":"https://openalex.org/F4320336350","funder_display_name":"Key Research and Development Projects of Shaanxi Province"},{"id":"https://openalex.org/G8008072069","display_name":null,"funder_award_id":"U24A20252","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320336350","display_name":"Key Research and Development Projects of Shaanxi Province","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7138914713.pdf","grobid_xml":"https://content.openalex.org/works/W7138914713.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multi-modal":[0],"Retrieval-Augmented":[1],"Generation":[2],"(MMRAG)":[3],"enables":[4],"highly":[5],"credible":[6],"generation":[7],"by":[8],"integrating":[9],"external":[10],"multi-modal":[11,19,54,62,75,94,123,149],"knowledge,":[12],"thus":[13],"demonstrating":[14],"impressive":[15],"performance":[16],"in":[17,79,132],"complex":[18],"scenarios.":[20],"However,":[21],"existing":[22],"MMRAG":[23,134],"methods":[24],"fail":[25],"to":[26,49,72,88,113,127],"clarify":[27],"the":[28,38,41,58,80,105,133],"reasoning":[29,59,130],"logic":[30,131],"behind":[31],"retrieval":[32],"and":[33,119,143,152],"response":[34],"generation,":[35,56,121,151],"which":[36],"limits":[37],"explainability":[39],"of":[40,61,93],"results.":[42],"To":[43],"address":[44],"this":[45],"gap,":[46],"we":[47],"propose":[48],"introduce":[50],"reinforcement":[51,69,84,109],"learning":[52],"into":[53],"retrieval-augmented":[55,76,150],"enhancing":[57],"capabilities":[60],"large":[63,124],"language":[64,125],"models":[65,126],"through":[66,157],"a":[67],"two-stage":[68],"fine-tuning":[70,85,110],"framework":[71],"achieve":[73],"explainable":[74,129],"generation.":[77],"Specifically,":[78],"first":[81],"stage,":[82,107],"rule-based":[83],"is":[86,111,155],"employed":[87],"perform":[89],"coarse-grained":[90],"point-wise":[91],"ranking":[92,118],"documents,":[95],"effectively":[96],"filtering":[97],"out":[98],"those":[99],"that":[100],"are":[101],"significantly":[102],"irrelevant.":[103],"In":[104],"second":[106],"reasoning-based":[108],"utilized":[112],"jointly":[114],"optimize":[115],"fine-grained":[116],"list-wise":[117],"answer":[120],"guiding":[122],"output":[128],"process.":[135],"Our":[136],"method":[137],"achieves":[138],"state-of-the-art":[139],"results":[140],"on":[141],"WebQA":[142],"MultimodalQA,":[144],"two":[145],"benchmark":[146],"datasets":[147],"for":[148],"its":[153],"effectiveness":[154],"validated":[156],"comprehensive":[158],"ablation":[159],"experiments.":[160]},"counts_by_year":[],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2026-03-20T00:00:00"}