{"id":"https://openalex.org/W4415329957","doi":"https://doi.org/10.52202/085713-1981","title":"What Makes a Reward Model a Good Teacher? An Optimization Perspective","display_name":"What Makes a Reward Model a Good Teacher? An Optimization Perspective","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4415329957","doi":"https://doi.org/10.52202/085713-1981"},"language":"en","primary_location":{"id":"doi:10.52202/085713-1981","doi":"https://doi.org/10.52202/085713-1981","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-1981","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2503.15477","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030945372","display_name":"Noam Razin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Noam Razin","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zixuan Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zixuan Wang","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hubert Strauss","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hubert Strauss","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110910362","display_name":"Stanley Wei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stanley Wei","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059740024","display_name":"Jason D. Lee","orcid":"https://orcid.org/0000-0003-0064-7800","observed_orcids":["https://orcid.org/0000-0003-0064-7800","https://orcid.org/0000-0002-7792-6659","https://orcid.org/0000-0003-1604-1395"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jason Lee","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103209777","display_name":"Sanjeev Arora","orcid":"https://orcid.org/0000-0002-8636-9970","observed_orcids":["https://orcid.org/0000-0002-8636-9970"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sanjeev Arora","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I20089843"],"apc_list":null,"apc_paid":null,"fwci":21.4393,"has_fulltext":true,"cited_by_count":59,"citation_normalized_percentile":{"value":0.99566648,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"65916","last_page":"65976"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9779999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9779999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reward-models","display_name":"reward models","score":0.7889999747276306},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.8930000066757202},{"id":"https://openalex.org/keywords/reward-variance","display_name":"reward variance","score":0.9670000076293945},{"id":"https://openalex.org/keywords/optimization-efficiency","display_name":"optimization efficiency","score":0.6150000095367432},{"id":"https://openalex.org/keywords/cross-model-transferability","display_name":"cross-model transfer","score":0.36800000071525574},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6240000128746033}],"concepts":[{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6355999708175659},{"id":"https://openalex.org/C2776330181","wikidata":"https://www.wikidata.org/wiki/Q18358244","display_name":"Maximization","level":2,"score":0.6326000094413757},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5590000152587891},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5565000176429749},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5404999852180481},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5113000273704529},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.38830000162124634},{"id":"https://openalex.org/C143661069","wikidata":"https://www.wikidata.org/wiki/Q670713","display_name":"Reward system","level":2,"score":0.38040000200271606},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3749000132083893},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3384999930858612},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2867000102996826}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-1981","doi":"https://doi.org/10.52202/085713-1981","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-1981","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.15477","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.15477","pdf_url":"https://arxiv.org/pdf/2503.15477","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2503.15477","doi":"https://doi.org/10.48550/arxiv.2503.15477","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.15477","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2503.15477","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.15477","pdf_url":"https://arxiv.org/pdf/2503.15477","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1479631004","display_name":"Collaborative Research: CIF: Medium: MoDL:Toward a Mathematical Foundation of Deep Reinforcement Learning","funder_award_id":"2212262","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3136253050","display_name":"CAREER: Towards a Theory of Deep Learning","funder_award_id":"2144994","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8429103732","display_name":"AI Institute: Institute for Foundations of Machine Learning","funder_award_id":"2019844","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8778260975","display_name":"RI: Medium: Provable Reinforcement Learning with Function Approximation and Neural Networks","funder_award_id":"2107304","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320315389","display_name":"Open Philanthropy Project","ror":"https://ror.org/004d1k391"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415329957.pdf","grobid_xml":"https://content.openalex.org/works/W4415329957.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W2462906003","https://openalex.org/W2736601468","https://openalex.org/W2940545298","https://openalex.org/W2946841987","https://openalex.org/W2971587637","https://openalex.org/W2973379954","https://openalex.org/W2980282514","https://openalex.org/W4223908421","https://openalex.org/W4226227340","https://openalex.org/W4366196653","https://openalex.org/W4378567238","https://openalex.org/W4384111857","https://openalex.org/W4385262413","https://openalex.org/W4386080611","https://openalex.org/W4387994989","https://openalex.org/W4391420971","https://openalex.org/W4391631327","https://openalex.org/W4391871179","https://openalex.org/W4394664305","https://openalex.org/W4394906585","https://openalex.org/W4396945613","https://openalex.org/W4401306886","https://openalex.org/W4402672014","https://openalex.org/W4404088829","https://openalex.org/W4404304546","https://openalex.org/W4404782997","https://openalex.org/W4404792842","https://openalex.org/W4404987838","https://openalex.org/W4405904217","https://openalex.org/W4406961529","https://openalex.org/W4411119916","https://openalex.org/W4414281281","https://openalex.org/W4414544662","https://openalex.org/W4416044876","https://openalex.org/W7197049846"],"related_works":["https://openalex.org/W4404782997","https://openalex.org/W4393063868","https://openalex.org/W4404303906","https://openalex.org/W4390833718","https://openalex.org/W4404782113","https://openalex.org/W4404304546","https://openalex.org/W4394664305","https://openalex.org/W7165375242","https://openalex.org/W4415277284","https://openalex.org/W4403853601"],"abstract_inverted_index":{"The":[0],"success":[1],"of":[2,14,55,132,142,151],"Reinforcement":[3],"Learning":[4],"from":[5,46,73],"Human":[6],"Feedback":[7],"(RLHF)":[8],"critically":[9],"depends":[10],"on":[11,138],"the":[12,15,69,143,160],"quality":[13,21],"reward":[16,37,59,66,82,97,104,116,134,163,167,178],"model.":[17],"However,":[18],"while":[19],"this":[20,44],"is":[22],"primarily":[23],"evaluated":[24],"through":[25],"accuracy,":[26,165,176],"it":[27,63],"remains":[28],"unclear":[29],"whether":[30],"accuracy":[31,139],"fully":[32],"captures":[33],"what":[34],"makes":[35],"a":[36,58,74,79,103,120,129,177],"model":[38,60,83,105,112,145,179],"an":[39,47],"effective":[40],"teacher.":[41],"We":[42,99],"address":[43],"question":[45],"optimization":[48],"perspective.":[49],"First,":[50],"we":[51],"prove":[52],"that":[53,94,102,106,174],"regardless":[54],"how":[56],"accurate":[57,81,92],"is,":[61],"if":[62],"induces":[64],"low":[65,115],"variance,":[67,117,164],"then":[68],"RLHF":[70],"objective":[71,122],"suffers":[72],"flat":[75,121],"landscape.":[76],"Consequently,":[77],"even":[78],"perfectly":[80],"can":[84,113],"lead":[85],"to":[86,153,181],"extremely":[87],"slow":[88],"optimization,":[89],"underperforming":[90],"less":[91],"models":[93,135,150],"induce":[95,114,182],"higher":[96],"variance.":[98],"additionally":[100],"show":[101],"works":[107],"well":[108],"for":[109,124,185],"one":[110],"language":[111,144],"and":[118,166],"thus":[119],"landscape,":[123],"another.":[125],"These":[126],"results":[127],"establish":[128],"fundamental":[130],"limitation":[131],"evaluating":[133],"solely":[136],"based":[137],"or":[140],"independently":[141],"they":[146],"guide.":[147],"Experiments":[148],"using":[149],"up":[152],"8B":[154],"parameters":[155],"corroborate":[156],"our":[157,171],"theory,":[158],"demonstrating":[159],"interplay":[161],"between":[162],"maximization":[168],"rate.":[169],"Overall,":[170],"findings":[172],"highlight":[173],"beyond":[175],"needs":[180],"sufficient":[183],"variance":[184],"efficient":[186],"optimization.":[187]},"counts_by_year":[{"year":2026,"cited_by_count":21},{"year":2025,"cited_by_count":38}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-19T00:00:00"}