{"id":"https://openalex.org/W6928748004","doi":"https://doi.org/10.3929/ethz-b-000651806","title":"Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback","display_name":"Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback","publication_year":2023,"publication_date":"2023-12-01","ids":{"openalex":"https://openalex.org/W6928748004","doi":"https://doi.org/10.3929/ethz-b-000651806"},"language":"en","primary_location":{"id":"pmh:oai:www.research-collection.ethz.ch:20.500.11850/651806","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.11850/651806","pdf_url":null,"source":{"id":"https://openalex.org/S4306402302","display_name":"Repository for Publications and Research Data (ETH Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I35440088","host_organization_name":"ETH Zurich","host_organization_lineage":["https://openalex.org/I35440088"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions on Machine Learning Research","raw_type":"info:eu-repo/semantics/article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://hdl.handle.net/20.500.11850/651806","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Casper, Stephen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Casper, Stephen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5145865735","display_name":"Xander Davies","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Davies, Xander","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Shi, Claudia","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Claudia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Krendl Gilbert, Thomas","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krendl Gilbert, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064744991","display_name":"J\u00e9r\u00e9my Scheurer","orcid":"https://orcid.org/0000-0002-6859-6029","observed_orcids":["https://orcid.org/0000-0002-6859-6029"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scheurer, J\u00e9r\u00e9my","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018893572","display_name":"Javier Rando Ram\u00edrez","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rando Ramirez, Javier","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078302436","display_name":"Rachel Freedman","orcid":"https://orcid.org/0000-0003-3299-4313","observed_orcids":["https://orcid.org/0000-0003-3299-4313"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Freedman, Rachel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002211679","display_name":"Tomasz Korbak","orcid":"https://orcid.org/0000-0002-6258-2013","observed_orcids":["https://orcid.org/0000-0002-6258-2013"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Korbak, Tomasz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059597480","display_name":"David Lindner","orcid":"https://orcid.org/0000-0001-7051-7433","observed_orcids":["https://orcid.org/0000-0001-7051-7433"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lindner, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Freire, Pedro","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Freire, Pedro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101508272","display_name":"Tony Wang","orcid":"https://orcid.org/0000-0002-6074-3127","observed_orcids":["https://orcid.org/0000-0002-6074-3127"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Tony","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Marks, Samuel","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marks, Samuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Segerie, Charbel-Rapha\u00ebl","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Segerie, Charbel-Rapha\u00ebl","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Carroll, Micah","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Carroll, Micah","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020921993","display_name":"Andi Peng","orcid":"https://orcid.org/0000-0001-8136-6175","observed_orcids":["https://orcid.org/0000-0001-8136-6175"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Andi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018045214","display_name":"Phillip J. K. Christoffersen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Christoffersen, Phillip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040083642","display_name":"Mehul Damani","orcid":"https://orcid.org/0000-0001-5790-376X","observed_orcids":["https://orcid.org/0000-0001-5790-376X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Damani, Mehul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026484447","display_name":"Stewart Slocum","orcid":"https://orcid.org/0000-0003-1740-3407","observed_orcids":["https://orcid.org/0000-0003-1740-3407"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Slocum, Stewart","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Anwar, Usman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anwar, Usman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077343544","display_name":"Anand Siththaranjan","orcid":"https://orcid.org/0000-0001-7679-0414","observed_orcids":["https://orcid.org/0000-0001-7679-0414"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Siththaranjan, Anand","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074918225","display_name":"Max Nadeau","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nadeau, Max","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071048251","display_name":"Eric J. Michaud","orcid":"https://orcid.org/0000-0001-7912-1953","observed_orcids":["https://orcid.org/0000-0001-7912-1953"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michaud, Eric J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010933763","display_name":"Jacob Pfau","orcid":"https://orcid.org/0000-0002-5771-8465","observed_orcids":["https://orcid.org/0000-0002-5771-8465"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pfau, Jacob","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053712506","display_name":"Dmitrii Krasheninnikov","orcid":"https://orcid.org/0009-0009-4387-8407","observed_orcids":["https://orcid.org/0009-0009-4387-8407"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krasheninnikov, Dmitrii","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Xin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020285189","display_name":"Lauro Langosco","orcid":"https://orcid.org/0000-0002-6868-5304","observed_orcids":["https://orcid.org/0000-0002-6868-5304"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Langosco, Lauro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092574633","display_name":"Peter Hase","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hase, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031426401","display_name":"Erdem B\u0131y\u0131k","orcid":"https://orcid.org/0000-0002-9516-3130","observed_orcids":["https://orcid.org/0000-0002-9516-3130"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"B\u0131y\u0131k, Erdem","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Dragan, Anca","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dragan, Anca","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029025914","display_name":"David Krueger","orcid":"https://orcid.org/0000-0001-7256-0937","observed_orcids":["https://orcid.org/0000-0001-7256-0937"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krueger, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080725225","display_name":"Dorsa Sadigh","orcid":"https://orcid.org/0000-0002-7802-9183","observed_orcids":["https://orcid.org/0000-0002-7802-9183"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sadigh, Dorsa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5076757561","display_name":"Dylan Hadfield-Menell","orcid":"https://orcid.org/0000-0002-6168-4763","observed_orcids":["https://orcid.org/0000-0002-6168-4763"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hadfield-Menell, Dylan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2848,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.54326988,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":96},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.9850000143051147},{"id":"https://openalex.org/keywords/fundamental-limits","display_name":"fundamental limits","score":0.7689999938011169},{"id":"https://openalex.org/keywords/open-problems","display_name":"open problems","score":0.9369999766349792},{"id":"https://openalex.org/keywords/human-feedback","display_name":"human feedback","score":0.550000011920929}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5636000037193298},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5497000217437744},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40799999237060547},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.3864000141620636},{"id":"https://openalex.org/C3018651601","wikidata":"https://www.wikidata.org/wiki/Q183635","display_name":"Feedback control","level":2,"score":0.3393999934196472},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.32280001044273376},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.30799999833106995},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.28619998693466187},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27090001106262207},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2565000057220459}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:www.research-collection.ethz.ch:20.500.11850/651806","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.11850/651806","pdf_url":null,"source":{"id":"https://openalex.org/S4306402302","display_name":"Repository for Publications and Research Data (ETH Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I35440088","host_organization_name":"ETH Zurich","host_organization_lineage":["https://openalex.org/I35440088"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions on Machine Learning Research","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.3929/ethz-b-000651806","doi":"https://doi.org/10.3929/ethz-b-000651806","is_oa":true,"landing_page_url":"https://doi.org/10.3929/ethz-b-000651806","pdf_url":null,"source":{"id":"https://openalex.org/S7407051236","display_name":"ETH Z\u00fcrich Research Collection","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:www.research-collection.ethz.ch:20.500.11850/651806","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.11850/651806","pdf_url":null,"source":{"id":"https://openalex.org/S4306402302","display_name":"Repository for Publications and Research Data (ETH Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I35440088","host_organization_name":"ETH Zurich","host_organization_lineage":["https://openalex.org/I35440088"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions on Machine Learning Research","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2796530402","https://openalex.org/W3108991798","https://openalex.org/W6950439104","https://openalex.org/W7045589309","https://openalex.org/W3017404084","https://openalex.org/W7204196492","https://openalex.org/W7151772826","https://openalex.org/W4389520781","https://openalex.org/W4415795359","https://openalex.org/W24473057"],"abstract_inverted_index":null,"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2025-10-10T00:00:00"}