{"id":"https://openalex.org/W4366196653","doi":"https://doi.org/10.48550/arxiv.2304.06767","title":"RAFT: Reward rAnked FineTuning for Generative Foundation Model Alignment","display_name":"RAFT: Reward rAnked FineTuning for Generative Foundation Model Alignment","publication_year":2023,"publication_date":"2023-04-13","ids":{"openalex":"https://openalex.org/W4366196653","doi":"https://doi.org/10.48550/arxiv.2304.06767"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2304.06767","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.06767","pdf_url":"https://arxiv.org/pdf/2304.06767","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2304.06767","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061778641","display_name":"Hanze Dong","orcid":"https://orcid.org/0000-0002-8846-1260","observed_orcids":["https://orcid.org/0000-0002-8846-1260"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Hanze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008776627","display_name":"Wei Hao Xiong","orcid":"https://orcid.org/0000-0002-2532-2503","observed_orcids":["https://orcid.org/0000-0002-2532-2503","https://orcid.org/0000-0003-3285-2067","https://orcid.org/0009-0008-0692-1296"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiong, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Goyal, Deepanshu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Goyal, Deepanshu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhang, Yihan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chow, Winnie","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chow, Winnie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100638027","display_name":"Rui Pan","orcid":"https://orcid.org/0000-0002-6973-3259","observed_orcids":["https://orcid.org/0000-0002-6973-3259"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058966093","display_name":"Shizhe Diao","orcid":"https://orcid.org/0000-0002-3325-9209","observed_orcids":["https://orcid.org/0000-0002-3325-9209"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Diao, Shizhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100622779","display_name":"Jipeng Zhang","orcid":"https://orcid.org/0000-0002-3269-6992","observed_orcids":["https://orcid.org/0000-0002-3269-6992","https://orcid.org/0000-0001-9638-2741"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050914724","display_name":"Kashun Shum","orcid":"https://orcid.org/0009-0006-5168-8345","observed_orcids":["https://orcid.org/0009-0006-5168-8345"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shum, Kashun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100378792","display_name":"Tong Zhang","orcid":"https://orcid.org/0000-0002-5511-2558","observed_orcids":["https://orcid.org/0000-0002-5511-2558","https://orcid.org/0000-0001-5818-4285","https://orcid.org/0000-0002-8838-4963","https://orcid.org/0000-0002-7025-6365","https://orcid.org/0000-0002-3506-0180","https://orcid.org/0000-0001-6212-4891","https://orcid.org/0000-0003-1148-4322","https://orcid.org/0000-0002-0131-0129","https://orcid.org/0000-0001-8163-3050","https://orcid.org/0000-0002-3481-6062"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Tong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":31,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.7490000128746033,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.7490000128746033,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.05009999871253967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.03970000147819519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/implicit-bias","display_name":"implicit bias","score":0.47999998927116394},{"id":"https://openalex.org/keywords/sample-filtering","display_name":"sample filtering","score":0.5210000276565552},{"id":"https://openalex.org/keywords/reward-learning","display_name":"reward learning","score":0.6959999799728394},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8610000014305115},{"id":"https://openalex.org/keywords/diffusion-models","display_name":"diffusion models","score":0.9909999966621399}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7794328927993774},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.7401258945465088},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6887614727020264},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6143700480461121},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6096891760826111},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.5942546725273132},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.5793092846870422},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2304.06767","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.06767","pdf_url":"https://arxiv.org/pdf/2304.06767","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-146198","doi":null,"is_oa":false,"landing_page_url":"http://www.scopus.com/record/display.url?eid=2-s2.0-86000102713&origin=inward","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2304.06767","doi":"https://doi.org/10.48550/arxiv.2304.06767","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2304.06767","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2304.06767","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.06767","pdf_url":"https://arxiv.org/pdf/2304.06767","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4365211920","https://openalex.org/W3014948380","https://openalex.org/W4380551139","https://openalex.org/W4317695495","https://openalex.org/W4387506531","https://openalex.org/W4238433571","https://openalex.org/W3174044702","https://openalex.org/W2967848559","https://openalex.org/W4299831724","https://openalex.org/W4283803360"],"abstract_inverted_index":{"Generative":[0],"foundation":[1],"models":[2,33,70,125,183],"are":[3,71],"susceptible":[4],"to":[5,64,95,122],"implicit":[6],"biases":[7,17],"that":[8,145,163],"can":[9,18,165],"arise":[10],"from":[11,60],"extensive":[12],"unsupervised":[13],"training":[14],"data.":[15],"Such":[16],"produce":[19],"suboptimal":[20],"samples,":[21,136,142],"skewed":[22],"outcomes,":[23],"and":[24,37,47,85,106,131,149,175,184],"unfairness,":[25],"with":[26,34,73,88],"potentially":[27],"serious":[28],"consequences.":[29],"Consequently,":[30],"aligning":[31],"these":[32,157],"human":[35],"ethics":[36],"preferences":[38],"is":[39],"an":[40],"essential":[41],"step":[42],"toward":[43],"ensuring":[44],"their":[45],"responsible":[46],"effective":[48],"deployment":[49],"in":[50,171,179],"real-world":[51],"applications.":[52],"Prior":[53],"research":[54],"has":[55],"primarily":[56],"employed":[57],"Reinforcement":[58],"Learning":[59],"Human":[61],"Feedback":[62],"(RLHF)":[63],"address":[65],"this":[66,110],"problem,":[67],"where":[68],"generative":[69,124],"fine-tuned":[72],"RL":[74,89],"algorithms":[75,90],"guided":[76],"by":[77,154],"a":[78,103,114,128,132],"human-feedback-informed":[79],"reward":[80,129,173],"model.":[81],"However,":[82],"the":[83,96,100,140,152,168],"inefficiencies":[84],"instabilities":[86],"associated":[87],"frequently":[91],"present":[92],"substantial":[93],"obstacles":[94],"successful":[97],"alignment,":[98],"necessitating":[99],"development":[101],"of":[102,135],"more":[104],"robust":[105],"streamlined":[107],"approach.":[108],"To":[109],"end,":[111],"we":[112],"introduce":[113],"new":[115],"framework,":[116],"Reward":[117],"rAnked":[118],"FineTuning":[119],"(RAFT),":[120],"designed":[121],"align":[123],"effectively.":[126],"Utilizing":[127],"model":[130,153,169],"sufficient":[133],"number":[134],"our":[137],"approach":[138],"selects":[139],"high-quality":[141],"discarding":[143],"those":[144],"exhibit":[146],"undesired":[147],"behavior,":[148],"subsequently":[150],"enhancing":[151],"fine-tuning":[155],"on":[156],"filtered":[158],"samples.":[159],"Our":[160],"studies":[161],"show":[162],"RAFT":[164],"effectively":[166],"improve":[167],"performance":[170],"both":[172,180],"learning":[174],"other":[176],"automated":[177],"metrics":[178],"large":[181],"language":[182],"diffusion":[185],"models.":[186]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":10}],"updated_date":"2026-10-13T10:08:12.033832","created_date":"2025-10-10T00:00:00"}