{"id":"https://openalex.org/W4411063470","doi":"https://doi.org/10.1145/3743127","title":"RLHF Deciphered: A Critical Analysis of Reinforcement Learning from Human Feedback for LLMs","display_name":"RLHF Deciphered: A Critical Analysis of Reinforcement Learning from Human Feedback for LLMs","publication_year":2025,"publication_date":"2025-06-05","ids":{"openalex":"https://openalex.org/W4411063470","doi":"https://doi.org/10.1145/3743127"},"language":"en","primary_location":{"id":"doi:10.1145/3743127","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3743127","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3743127","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091446143","display_name":"Shreyas Chaudhari","orcid":"https://orcid.org/0000-0002-8826-2253"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shreyas Chaudhari","raw_affiliation_strings":["University of Massachusetts Amherst","University of Massachusetts Amherst,  Amherst, United States"],"raw_orcid":"https://orcid.org/0000-0002-8826-2253","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst","institution_ids":["https://openalex.org/I24603500"]},{"raw_affiliation_string":"University of Massachusetts Amherst,  Amherst, United States","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045442167","display_name":"Pranjal Aggarwal","orcid":"https://orcid.org/0000-0002-2962-1535"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pranjal Aggarwal","raw_affiliation_strings":["Carnegie Mellon University","Carnegie Mellon University,  Pittsburgh, United States"],"raw_orcid":"https://orcid.org/0000-0002-2962-1535","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University,  Pittsburgh, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049841785","display_name":"Vishvak Murahari","orcid":null},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vishvak Murahari","raw_affiliation_strings":["Princeton University","Princeton University,  Princeton, United States"],"raw_orcid":"https://orcid.org/0009-0008-9304-0701","affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Princeton University,  Princeton, United States","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019768140","display_name":"Tanmay Rajpurohit","orcid":"https://orcid.org/0000-0001-9302-4244"},"institutions":[{"id":"https://openalex.org/I2802723755","display_name":"Independent Sector","ror":"https://ror.org/05vhwqa91","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I2802723755"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tanmay Rajpurohit","raw_affiliation_strings":["Independent Researcher","Independent Researcher,  Seattle, United States"],"raw_orcid":"https://orcid.org/0000-0001-9302-4244","affiliations":[{"raw_affiliation_string":"Independent Researcher","institution_ids":[]},{"raw_affiliation_string":"Independent Researcher,  Seattle, United States","institution_ids":["https://openalex.org/I2802723755","https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114053294","display_name":"Ashwin Kalyan","orcid":null},"institutions":[{"id":"https://openalex.org/I2802723755","display_name":"Independent Sector","ror":"https://ror.org/05vhwqa91","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I2802723755"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ashwin Kalyan","raw_affiliation_strings":["Independent Researcher","Independent Researcher,  Seattle, United States"],"raw_orcid":"https://orcid.org/0009-0006-3448-3305","affiliations":[{"raw_affiliation_string":"Independent Researcher","institution_ids":[]},{"raw_affiliation_string":"Independent Researcher,  Seattle, United States","institution_ids":["https://openalex.org/I2802723755","https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025205227","display_name":"Karthik Narasimhan","orcid":null},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Karthik Narasimhan","raw_affiliation_strings":["Princeton University","Princeton University,  Princeton, United States"],"raw_orcid":"https://orcid.org/0000-0001-9894-9983","affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Princeton University,  Princeton, United States","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041777363","display_name":"Ameet Deshpande","orcid":"https://orcid.org/0000-0001-9885-0385"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ameet Deshpande","raw_affiliation_strings":["Princeton University","Princeton University,  Princeton, United States"],"raw_orcid":"https://orcid.org/0000-0001-9885-0385","affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Princeton University,  Princeton, United States","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034701757","display_name":"Bruno Castro da Silva","orcid":"https://orcid.org/0000-0002-3708-5728"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bruno Castro da Silva","raw_affiliation_strings":["University of Massachusetts Amherst","University of Massachusetts Amherst,  Amherst, United States"],"raw_orcid":"https://orcid.org/0000-0002-3708-5728","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst","institution_ids":["https://openalex.org/I24603500"]},{"raw_affiliation_string":"University of Massachusetts Amherst,  Amherst, United States","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5091446143"],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":79.001,"has_fulltext":false,"cited_by_count":41,"citation_normalized_percentile":{"value":0.99933653,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"58","issue":"2","first_page":"1","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.9822999835014343,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8306971788406372},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7492228746414185},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3670487105846405},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3298098146915436}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8306971788406372},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7492228746414185},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3670487105846405},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3298098146915436}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3743127","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3743127","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3743127","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3743127","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W43928053","https://openalex.org/W1592847719","https://openalex.org/W1837766154","https://openalex.org/W1994306835","https://openalex.org/W1995500779","https://openalex.org/W2013784666","https://openalex.org/W2101524054","https://openalex.org/W2107726111","https://openalex.org/W2121863487","https://openalex.org/W2525579820","https://openalex.org/W2560398012","https://openalex.org/W2803933017","https://openalex.org/W2963567240","https://openalex.org/W3100789280","https://openalex.org/W3164005523","https://openalex.org/W3208954537","https://openalex.org/W4205326910","https://openalex.org/W4212863985","https://openalex.org/W4214717370","https://openalex.org/W4221152344","https://openalex.org/W4223477065","https://openalex.org/W4226126941","https://openalex.org/W4232035701","https://openalex.org/W4234333737","https://openalex.org/W4285210452","https://openalex.org/W4300771675","https://openalex.org/W4384071683","https://openalex.org/W4385572830","https://openalex.org/W4385632485","https://openalex.org/W4389519204","https://openalex.org/W4389762782","https://openalex.org/W4390430574","https://openalex.org/W4408507277","https://openalex.org/W6859606847"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4306904969","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2138720691","https://openalex.org/W2376932109"],"abstract_inverted_index":{"A":[0,79],"significant":[1],"challenge":[2],"in":[3],"training":[4],"large":[5],"language":[6],"models":[7],"(LLMs)":[8],"as":[9,26],"effective":[10],"assistants":[11],"is":[12,35],"aligning":[13],"them":[14],"with":[15],"human":[16,21],"preferences.":[17],"Reinforcement":[18],"learning":[19,48],"from":[20],"feedback":[22],"(RLHF)":[23],"has":[24],"emerged":[25],"a":[27],"promising":[28],"solution.":[29],"However,":[30],"our":[31],"understanding":[32],"of":[33,82,93],"RLHF":[34,45,94],"often":[36],"limited":[37],"to":[38,89],"initial":[39],"design":[40],"choices.":[41],"This":[42],"article":[43],"analyzes":[44],"through":[46],"reinforcement":[47],"principles,":[49],"focusing":[50],"on":[51],"the":[52,91],"reward":[53,66],"model.":[54],"It":[55],"examines":[56],"modeling":[57],"choices":[58],"and":[59,68,76,95],"function":[60],"approximation":[61],"caveats,":[62],"highlighting":[63],"assumptions":[64],"about":[65],"expressivity":[67],"revealing":[69],"limitations":[70],"like":[71],"incorrect":[72],"generalization,":[73],"model":[74],"misspecification,":[75],"sparse":[77],"feedback.":[78],"categorical":[80],"review":[81],"current":[83],"literature":[84],"provides":[85],"insights":[86],"for":[87],"researchers":[88],"understand":[90],"challenges":[92],"build":[96],"upon":[97],"existing":[98],"methods.":[99]},"counts_by_year":[{"year":2026,"cited_by_count":17},{"year":2025,"cited_by_count":24}],"updated_date":"2026-05-21T09:19:25.381259","created_date":"2025-10-10T00:00:00"}