{"id":"https://openalex.org/W7155016561","doi":"https://doi.org/10.48550/arxiv.2604.16242","title":"Detecting and Suppressing Reward Hacking with Gradient Fingerprints","display_name":"Detecting and Suppressing Reward Hacking with Gradient Fingerprints","publication_year":2026,"publication_date":"2026-04-17","ids":{"openalex":"https://openalex.org/W7155016561","doi":"https://doi.org/10.48550/arxiv.2604.16242"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.16242","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16242","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.16242","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134065645","display_name":"Songtao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Songtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008421532","display_name":"Quang Hi\u1ebfu Ph\u1ea1m","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pham, Quang Hieu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134015369","display_name":"Fangcong Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Fangcong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134057657","display_name":"Xinpeng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xinpeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134090098","display_name":"Jocelyn Qiaochu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jocelyn Qiaochu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134021991","display_name":"Greg Durrett","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Durrett, Greg","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134049942","display_name":"Xi Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Xi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.25850000977516174,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.25850000977516174,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.16920000314712524,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.08250000327825546,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.6021000146865845},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5770000219345093},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.5440999865531921},{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.5419999957084656},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5299999713897705},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.4909999966621399},{"id":"https://openalex.org/keywords/hacker","display_name":"Hacker","score":0.48100000619888306},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.45989999175071716}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7651000022888184},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.6021000146865845},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5770000219345093},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5537999868392944},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.5440999865531921},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.5419999957084656},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5299999713897705},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4909999966621399},{"id":"https://openalex.org/C86844869","wikidata":"https://www.wikidata.org/wiki/Q2798820","display_name":"Hacker","level":2,"score":0.48100000619888306},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.45989999175071716},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44760000705718994},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.4465000033378601},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.38580000400543213},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.29089999198913574},{"id":"https://openalex.org/C40305131","wikidata":"https://www.wikidata.org/wiki/Q2616305","display_name":"Obfuscation","level":2,"score":0.2761000096797943},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2678000032901764},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C2777826928","wikidata":"https://www.wikidata.org/wiki/Q3745713","display_name":"Fingerprint (computing)","level":2,"score":0.2615000009536743},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.2596000134944916},{"id":"https://openalex.org/C184297639","wikidata":"https://www.wikidata.org/wiki/Q177765","display_name":"Biometrics","level":2,"score":0.2583000063896179}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.16242","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16242","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.16242","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.16242","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"with":[2],"verifiable":[3,124],"rewards":[4,10],"(RLVR)":[5],"typically":[6],"optimizes":[7],"for":[8,78,161,186],"outcome":[9],"without":[11,42],"imposing":[12],"constraints":[13],"on":[14,61,100,170],"intermediate":[15,55],"reasoning.":[16],"This":[17],"leaves":[18],"training":[19,32],"susceptible":[20],"to":[21,38,114],"reward":[22,36,80,120,150,165],"hacking,":[23],"where":[24],"models":[25],"exploit":[26],"loopholes":[27],"(e.g.,":[28],"spurious":[29],"patterns":[30],"in":[31,34,148],"data)":[33],"the":[35,44,54,62,65,97,101,117,157,171,188],"function":[37],"achieve":[39],"high":[40],"scores":[41],"solving":[43],"intended":[45],"task.":[46],"These":[47],"reward-hacking":[48],"behaviors":[49],"are":[50],"often":[51],"implicit,":[52],"as":[53],"chain-of-thought":[56],"(CoT)":[57],"may":[58],"appear":[59],"plausible":[60],"surface,":[63],"limiting":[64],"effectiveness":[66],"of":[67,96,181,190],"purely":[68],"text-based":[69],"monitoring.":[70],"We":[71],"propose":[72],"Gradient":[73],"Fingerprint":[74],"(GRIFT),":[75],"a":[76,87,90,107,178],"method":[77],"detecting":[79,149],"hacking":[81,121,151,166],"using":[82],"models'":[83],"internal":[84],"computations.":[85],"Given":[86],"prompt":[88,102],"and":[89,103,130,141,167],"model-generated":[91],"CoT,":[92],"GRIFT":[93,133,155],"computes":[94],"gradients":[95],"CoT":[98,118,139,191],"conditioned":[99],"compresses":[104],"them":[105],"into":[106,156],"compact":[108],"representation,":[109],"which":[110],"is":[111,196],"then":[112],"used":[113],"assess":[115],"whether":[116],"reflects":[119],"behavior.":[122,152],"Across":[123],"reasoning":[125,162,192],"benchmarks":[126],"spanning":[127],"math,":[128],"code,":[129],"logical":[131],"reasoning,":[132],"substantially":[134],"outperforms":[135],"strong":[136],"baselines,":[137],"including":[138],"Monitor":[140],"TRACE,":[142],"achieving":[143],"over":[144],"25%":[145],"relative":[146],"improvement":[147],"Moreover,":[153],"integrating":[154],"rejection":[158],"fine-tuning":[159],"pipeline":[160],"tasks":[163],"reduces":[164],"improves":[168],"performance":[169],"true":[172],"task":[173],"objective.":[174],"Our":[175,194],"results":[176],"highlight":[177],"promising":[179],"direction":[180],"leveraging":[182],"gradient":[183],"level":[184],"representations":[185],"assessing":[187],"quality":[189],"traces.":[193],"code":[195],"available":[197],"at:":[198],"https://github.com/songtao-x/reward_hack.":[199]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-21T00:00:00"}