{"id":"https://openalex.org/W4396914049","doi":"https://doi.org/10.48550/arxiv.2405.07863","title":"RLHF Workflow: From Reward Modeling to Online RLHF","display_name":"RLHF Workflow: From Reward Modeling to Online RLHF","publication_year":2024,"publication_date":"2024-05-13","ids":{"openalex":"https://openalex.org/W4396914049","doi":"https://doi.org/10.48550/arxiv.2405.07863"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.07863","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.07863","pdf_url":"https://arxiv.org/pdf/2405.07863","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.07863","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061778641","display_name":"Hanze Dong","orcid":"https://orcid.org/0000-0002-8846-1260","observed_orcids":["https://orcid.org/0000-0002-8846-1260"]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dong, Hanze","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101382803","display_name":"Wei Xiong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiong, Wei","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112515039","display_name":"Bo Pang","orcid":"https://orcid.org/0009-0000-7182-1832","observed_orcids":["https://orcid.org/0009-0000-7182-1832"]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pang, Bo","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042417097","display_name":"Haoxiang Wang","orcid":"https://orcid.org/0000-0002-0361-6527","observed_orcids":["https://orcid.org/0000-0002-0361-6527","https://orcid.org/0000-0002-8587-3024"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Haoxiang","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100575498","display_name":"Han Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhao, Han","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhou, Yingbo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhou, Yingbo","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022613439","display_name":"Nan Jiang","orcid":"https://orcid.org/0000-0003-1581-0180","observed_orcids":["https://orcid.org/0000-0003-1581-0180"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiang, Nan","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101521933","display_name":"Doyen Sahoo","orcid":"https://orcid.org/0000-0003-4367-5852","observed_orcids":["https://orcid.org/0000-0003-4367-5852","https://orcid.org/0000-0003-3964-835X"]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sahoo, Doyen","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032046813","display_name":"Caiming Xiong","orcid":"https://orcid.org/0000-0003-0349-8628","observed_orcids":["https://orcid.org/0000-0003-0349-8628"]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiong, Caiming","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100378792","display_name":"Tong Zhang","orcid":"https://orcid.org/0000-0002-5511-2558","observed_orcids":["https://orcid.org/0000-0002-5511-2558","https://orcid.org/0000-0001-5818-4285","https://orcid.org/0000-0002-8838-4963","https://orcid.org/0000-0002-7025-6365","https://orcid.org/0000-0002-3506-0180","https://orcid.org/0000-0001-6212-4891","https://orcid.org/0000-0002-6807-0641","https://orcid.org/0000-0003-1148-4322","https://orcid.org/0000-0002-0131-0129","https://orcid.org/0000-0001-8163-3050","https://orcid.org/0000-0002-3481-6062"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Tong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.38280001282691956,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.38280001282691956,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.16169999539852142,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.14650000631809235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reward-modeling","display_name":"reward modeling","score":0.8560000061988831},{"id":"https://openalex.org/keywords/preference-models","display_name":"preference models","score":0.7329999804496765},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7710000276565552},{"id":"https://openalex.org/keywords/supervised-fine-tuning","display_name":"supervised fine-tuning","score":0.8309999704360962},{"id":"https://openalex.org/keywords/open-source-dataset","display_name":"open-source dataset","score":0.7599999904632568},{"id":"https://openalex.org/keywords/mt-bench","display_name":"MT-Bench","score":0.9729999899864197},{"id":"https://openalex.org/keywords/humaneval","display_name":"HumanEval","score":0.9179999828338623},{"id":"https://openalex.org/keywords/truthfulqa","display_name":"TruthfulQA","score":0.9150000214576721}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.7495911121368408},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5180513858795166},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3765145540237427},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.20367908477783203}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.07863","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.07863","pdf_url":"https://arxiv.org/pdf/2405.07863","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.07863","doi":"https://doi.org/10.48550/arxiv.2405.07863","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.07863","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.07863","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.07863","pdf_url":"https://arxiv.org/pdf/2405.07863","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4396914049.pdf","grobid_xml":"https://content.openalex.org/works/W4396914049.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7113260674","https://openalex.org/W7196964931","https://openalex.org/W4408595355","https://openalex.org/W4414681422","https://openalex.org/W4417097328","https://openalex.org/W4390833718","https://openalex.org/W6959934489","https://openalex.org/W4365211596","https://openalex.org/W4387560783","https://openalex.org/W4404303906"],"abstract_inverted_index":{"We":[0,161],"present":[1],"the":[2,31,48,107,119],"workflow":[3],"of":[4,102],"Online":[5],"Iterative":[6],"Reinforcement":[7],"Learning":[8],"from":[9],"Human":[10],"Feedback":[11],"(RLHF)":[12],"in":[13,30,60],"this":[14,53,61],"technical":[15,54],"report,":[16,55],"which":[17],"is":[18,69,83],"widely":[19],"reported":[20],"to":[21,47,58,71,112,196],"outperform":[22],"its":[23],"offline":[24,49],"counterpart":[25],"by":[26,94,130],"a":[27,65,99,131],"large":[28,33],"margin":[29],"recent":[32],"language":[34],"model":[35,111],"(LLM)":[36],"literature.":[37],"However,":[38],"existing":[39],"open-source":[40,87,103,177],"RLHF":[41,170],"projects":[42],"are":[43],"still":[44],"largely":[45],"confined":[46],"learning":[50],"setting.":[51],"In":[52,77],"we":[56,92,117,180],"aim":[57],"fill":[59],"gap":[62],"and":[63,105,122,148,159,168,187,198],"provide":[64],"detailed":[66,132,202],"recipe":[67],"that":[68,164],"easy":[70],"reproduce":[72],"for":[73,86,200],"online":[74,80,126],"iterative":[75,127,169],"RLHF.":[76],"particular,":[78],"since":[79],"human":[81,114],"feedback":[82],"usually":[84],"infeasible":[85],"communities":[88],"with":[89,175],"limited":[90],"resources,":[91],"start":[93],"constructing":[95],"preference":[96,110],"models":[97],"using":[98],"diverse":[100],"set":[101],"datasets":[104],"use":[106],"constructed":[108],"proxy":[109],"approximate":[113],"feedback.":[115],"Then,":[116],"discuss":[118],"theoretical":[120],"insights":[121],"algorithmic":[123],"principles":[124],"behind":[125],"RLHF,":[128],"followed":[129],"practical":[133],"implementation.":[134],"Our":[135],"trained":[136],"LLM":[137,142],"achieves":[138],"impressive":[139],"performance":[140,174],"on":[141],"chatbot":[143],"benchmarks,":[144],"including":[145],"AlpacaEval-2,":[146],"Arena-Hard,":[147],"MT-Bench,":[149],"as":[150,152,157],"well":[151],"other":[153],"academic":[154],"benchmarks":[155],"such":[156],"HumanEval":[158],"TruthfulQA.":[160],"have":[162,181],"shown":[163],"supervised":[165],"fine-tuning":[166],"(SFT)":[167],"can":[171],"obtain":[172],"state-of-the-art":[173],"fully":[176],"datasets.":[178],"Further,":[179],"made":[182],"our":[183],"models,":[184],"curated":[185],"datasets,":[186],"comprehensive":[188],"step-by-step":[189],"code":[190],"guidebooks":[191],"publicly":[192],"available.":[193],"Please":[194],"refer":[195],"https://github.com/RLHFlow/RLHF-Reward-Modeling":[197],"https://github.com/RLHFlow/Online-RLHF":[199],"more":[201],"information.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2024-05-15T00:00:00"}