{"id":"https://openalex.org/W7151036606","doi":"https://doi.org/10.48550/arxiv.2604.02986","title":"Mitigating Reward Hacking in RLHF via Advantage Sign Robustness","display_name":"Mitigating Reward Hacking in RLHF via Advantage Sign Robustness","publication_year":2026,"publication_date":"2026-04-03","ids":{"openalex":"https://openalex.org/W7151036606","doi":"https://doi.org/10.48550/arxiv.2604.02986"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.02986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.02986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.02986","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133048012","display_name":"Shinnosuke Ono","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ono, Shinnosuke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070927069","display_name":"Johannes Ackermann","orcid":"https://orcid.org/0000-0001-7141-8912"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ackermann, Johannes","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091587258","display_name":"Soichiro Nishimori","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nishimori, Soichiro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133024989","display_name":"Takashi Ishida","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ishida, Takashi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133012578","display_name":"Masashi Sugiyama","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sugiyama, Masashi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5604000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5604000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.27619999647140503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.023399999365210533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.47209998965263367},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4607999920845032},{"id":"https://openalex.org/keywords/hacker","display_name":"Hacker","score":0.4348999857902527},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.4081000089645386},{"id":"https://openalex.org/keywords/perturbation","display_name":"Perturbation (astronomy)","score":0.4074999988079071},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.35040000081062317},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.3472000062465668}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6798999905586243},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.47209998965263367},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4607999920845032},{"id":"https://openalex.org/C86844869","wikidata":"https://www.wikidata.org/wiki/Q2798820","display_name":"Hacker","level":2,"score":0.4348999857902527},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.4081000089645386},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.4074999988079071},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37790000438690186},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3626999855041504},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.35040000081062317},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3472000062465668},{"id":"https://openalex.org/C97399411","wikidata":"https://www.wikidata.org/wiki/Q825367","display_name":"Coin flipping","level":2,"score":0.32919999957084656},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.28870001435279846},{"id":"https://openalex.org/C38858127","wikidata":"https://www.wikidata.org/wiki/Q5441228","display_name":"Feed forward","level":2,"score":0.2768000066280365},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.2768000066280365},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.26510000228881836},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2637999951839447},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25360000133514404},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.02986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.02986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.02986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.02986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.4110243320465088}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reward":[0],"models":[1],"(RMs)":[2],"used":[3],"in":[4,66,105],"reinforcement":[5],"learning":[6],"from":[7],"human":[8],"feedback":[9],"(RLHF)":[10],"are":[11],"vulnerable":[12],"to":[13,58,119],"reward":[14,34,160],"hacking:":[15],"as":[16],"the":[17,31,46,56,67,80,86,106,120,131,137],"policy":[18,90,107,132],"maximizes":[19],"a":[20,49,52,74,152],"learned":[21],"proxy":[22],"reward,":[23],"true":[24],"quality":[25],"plateaus":[26],"or":[27,117],"degrades.":[28],"We":[29],"make":[30],"assumption":[32],"that":[33,83,113],"hacking":[35],"is":[36,79,125],"often":[37],"caused":[38],"by":[39],"flipped":[40,53],"advantage":[41,87],"signs:":[42],"instead":[43],"of":[44,48],"reducing":[45],"likelihood":[47],"bad":[50],"response,":[51],"sign":[54,88],"causes":[55],"update":[57],"increase":[59],"it.":[60],"By":[61],"considering":[62],"an":[63],"adversarial":[64],"perturbation":[65,82],"RM":[68,121,138],"parameter":[69],"space,":[70],"we":[71,96],"can":[72,84],"derive":[73],"certified":[75],"sign-preservation":[76],"radius,":[77],"which":[78],"smallest":[81],"flip":[85],"during":[89],"optimization.":[91],"Based":[92],"on":[93],"this":[94],"formulation,":[95],"propose":[97],"Sign-Certified":[98],"Policy":[99],"Optimization":[100],"(SignCert-PO),":[101],"down-weighting":[102],"non-robust":[103],"completions":[104],"gradient":[108],"update.":[109],"Unlike":[110],"prior":[111],"approaches":[112],"require":[114],"multiple":[115],"RMs":[116],"access":[118],"training":[122],"data,":[123],"SignCert-PO":[124,149],"lightweight":[126],"and":[127,140,146,158],"operates":[128],"purely":[129],"at":[130],"optimization":[133],"stage":[134],"using":[135],"only":[136],"parameters":[139],"on-policy":[141],"completions.":[142],"On":[143],"TL;DR":[144],"summarization":[145],"AlpacaFarm":[147],"benchmarks,":[148],"consistently":[150],"achieves":[151],"better":[153],"win":[154],"rate":[155],"than":[156],"baselines":[157],"reduces":[159],"hacking.":[161]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-07T00:00:00"}