{"id":"https://openalex.org/W4387113775","doi":"https://doi.org/10.48550/arxiv.2309.14525","title":"Aligning Large Multimodal Models with Factually Augmented RLHF","display_name":"Aligning Large Multimodal Models with Factually Augmented RLHF","publication_year":2023,"publication_date":"2023-09-25","ids":{"openalex":"https://openalex.org/W4387113775","doi":"https://doi.org/10.48550/arxiv.2309.14525"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2309.14525","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.14525","pdf_url":"https://arxiv.org/pdf/2309.14525","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2309.14525","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020049212","display_name":"Zhiqing Sun","orcid":"https://orcid.org/0000-0003-1933-496X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Zhiqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100784815","display_name":"Sheng Shen","orcid":"https://orcid.org/0000-0002-8773-6365"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Sheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050360449","display_name":"Shengcao Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Shengcao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103047904","display_name":"Haotian Liu","orcid":"https://orcid.org/0000-0002-9554-7839"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Haotian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107893340","display_name":"Chunyuan Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Chunyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073742611","display_name":"Yikang Shen","orcid":"https://orcid.org/0000-0001-6836-0510"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Yikang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040877128","display_name":"Chuang Gan","orcid":"https://orcid.org/0000-0003-4031-5886"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gan, Chuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016962709","display_name":"Liang-Yan Gui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gui, Liang-Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039231803","display_name":"Yu-Xiong Wang","orcid":"https://orcid.org/0000-0003-1195-2739"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yu-Xiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041434096","display_name":"Yiming Yang","orcid":"https://orcid.org/0000-0003-1359-0364"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047285420","display_name":"Kurt Keutzer","orcid":"https://orcid.org/0000-0003-3868-8501"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Keutzer, Kurt","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5029105520","display_name":"Trevor Darrell","orcid":"https://orcid.org/0000-0001-5453-8533"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Darrell, Trevor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9872000217437744,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9090999960899353,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8259165287017822},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.672908365726471},{"id":"https://openalex.org/keywords/hallucinating","display_name":"Hallucinating","score":0.6693086624145508},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6499162912368774},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.584470808506012},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5716222524642944},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5344257950782776},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5302462577819824},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5256518125534058},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5226737856864929},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5180097222328186},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4699830114841461},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.44722381234169006},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3258236050605774}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8259165287017822},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.672908365726471},{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.6693086624145508},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6499162912368774},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.584470808506012},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5716222524642944},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5344257950782776},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5302462577819824},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5256518125534058},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5226737856864929},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5180097222328186},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4699830114841461},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.44722381234169006},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3258236050605774},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2309.14525","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.14525","pdf_url":"https://arxiv.org/pdf/2309.14525","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2309.14525","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2309.14525","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2309.14525","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.14525","pdf_url":"https://arxiv.org/pdf/2309.14525","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"score":0.41999998688697815,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387113775.pdf","grobid_xml":"https://content.openalex.org/works/W4387113775.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3204676188","https://openalex.org/W2992803471","https://openalex.org/W2066036438","https://openalex.org/W2078454190","https://openalex.org/W2963350370","https://openalex.org/W2930227984","https://openalex.org/W2108123836","https://openalex.org/W2063248114","https://openalex.org/W2528648655","https://openalex.org/W2520145460"],"abstract_inverted_index":{"Large":[0],"Multimodal":[1],"Models":[2],"(LMM)":[3],"are":[4,22,59],"built":[5],"across":[6],"modalities":[7,13],"and":[8,65,71,106,118,205],"the":[9,26,33,39,47,51,67,72,79,95,112,121,126,142,150,171,183,187,192,202],"misalignment":[10,35],"between":[11],"two":[12,63],"can":[14,199],"result":[15],"in":[16,29,116,153],"\"hallucination\",":[17],"generating":[18],"textual":[19],"outputs":[20],"that":[21,93],"not":[23],"grounded":[24],"by":[25,208],"multimodal":[27,34],"information":[28,101],"context.":[30],"To":[31,148],"address":[32],"issue,":[36],"we":[37,156],"adapt":[38],"Reinforcement":[40],"Learning":[41],"from":[42,46],"Human":[43],"Feedback":[44],"(RLHF)":[45],"text":[48],"domain":[49],"to":[50,61,77,140],"task":[52],"of":[53,145,191],"vision-language":[54,73],"alignment,":[55],"where":[56],"human":[57,81],"annotators":[58],"asked":[60],"compare":[62],"responses":[64],"pinpoint":[66],"more":[68],"hallucinated":[69],"one,":[70],"model":[74,97],"is":[75],"trained":[76,174],"maximize":[78],"simulated":[80],"rewards.":[82],"We":[83,123,215],"propose":[84],"a":[85,158,164],"new":[86,159],"alignment":[87],"algorithm":[88],"called":[89],"Factually":[90],"Augmented":[91],"RLHF":[92,117],"augments":[94],"reward":[96,113],"with":[98,134,163,175,186],"additional":[99],"factual":[100],"such":[102],"as":[103],"image":[104],"captions":[105],"ground-truth":[107],"multi-choice":[108],"options,":[109],"which":[110],"alleviates":[111],"hacking":[114],"phenomenon":[115],"further":[119],"improves":[120],"performance.":[122],"also":[124],"enhance":[125],"GPT-4-generated":[127],"training":[128],"data":[129,220],"(for":[130],"vision":[131],"instruction":[132],"tuning)":[133],"previously":[135],"available":[136],"human-written":[137],"image-text":[138],"pairs":[139],"improve":[141],"general":[143],"capabilities":[144],"our":[146,177,217],"model.":[147],"evaluate":[149],"proposed":[151],"approach":[152,178],"real-world":[154],"scenarios,":[155],"develop":[157],"evaluation":[160],"benchmark":[161],"MMHAL-BENCH":[162,211],"special":[165],"focus":[166],"on":[167,182,210],"penalizing":[168],"hallucinations.":[169],"As":[170],"first":[172],"LMM":[173],"RLHF,":[176],"achieves":[179],"remarkable":[180],"improvement":[181,207],"LLaVA-Bench":[184],"dataset":[185],"94%":[188],"performance":[189],"level":[190],"text-only":[193],"GPT-4":[194],"(while":[195],"previous":[196],"best":[197],"methods":[198],"only":[200],"achieve":[201],"87%":[203],"level),":[204],"an":[206],"60%":[209],"over":[212],"other":[213],"baselines.":[214],"opensource":[216],"code,":[218],"model,":[219],"at":[221],"https://llava-rlhf.github.io.":[222]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}