{"id":"https://openalex.org/W7164019727","doi":"https://doi.org/10.48550/arxiv.2606.09711","title":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","display_name":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","publication_year":2026,"publication_date":"2026-06-08","ids":{"openalex":"https://openalex.org/W7164019727","doi":"https://doi.org/10.48550/arxiv.2606.09711"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.09711","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09711","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.09711","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5124931005","display_name":"Mohammad Beigi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Beigi, Mohammad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138269719","display_name":"Ming Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Ming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138226594","display_name":"Lifu Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Lifu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.18799999356269836,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.18799999356269836,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.15929999947547913,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.15000000596046448,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.8032000064849854},{"id":"https://openalex.org/keywords/internalization","display_name":"Internalization","score":0.6972000002861023},{"id":"https://openalex.org/keywords/prime","display_name":"Prime (order theory)","score":0.5450999736785889},{"id":"https://openalex.org/keywords/hacker","display_name":"Hacker","score":0.5264000296592712},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.3846000134944916},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.38449999690055847}],"concepts":[{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.8032000064849854},{"id":"https://openalex.org/C139770010","wikidata":"https://www.wikidata.org/wiki/Q1339807","display_name":"Internalization","level":3,"score":0.6972000002861023},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5554999709129333},{"id":"https://openalex.org/C184992742","wikidata":"https://www.wikidata.org/wiki/Q7243229","display_name":"Prime (order theory)","level":2,"score":0.5450999736785889},{"id":"https://openalex.org/C86844869","wikidata":"https://www.wikidata.org/wiki/Q2798820","display_name":"Hacker","level":2,"score":0.5264000296592712},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39259999990463257},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.3846000134944916},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.38449999690055847},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32249999046325684},{"id":"https://openalex.org/C97508593","wikidata":"https://www.wikidata.org/wiki/Q1359069","display_name":"Contingency","level":2,"score":0.2831000089645386},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2784999907016754},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C191795146","wikidata":"https://www.wikidata.org/wiki/Q3878446","display_name":"Norm (philosophy)","level":2,"score":0.2596000134944916},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2563000023365021}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.09711","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09711","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.09711","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09711","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reward":[0,35],"hacking":[1],"is":[2,108],"usually":[3],"studied":[4],"after":[5],"it":[6],"becomes":[7],"visible,":[8],"once":[9],"a":[10,41,83,156,165],"model":[11],"earns":[12],"high":[13],"proxy":[14,25,49,153],"reward":[15,88,129],"while":[16],"failing":[17],"the":[18,104,115],"intended":[19],"task.":[20],"We":[21,32,77],"instead":[22],"study":[23],"what":[24],"RL":[26,59,154],"teaches":[27],"before":[28,86],"that":[29,79,91,151],"failure":[30],"appears.":[31],"introduce":[33],"Proxy":[34],"Internalization":[36],"and":[37,51,73,90,100,125,133],"Mechanistic":[38],"Exploitation":[39],"(PRIME),":[40],"learned":[42],"capability":[43,158],"to":[44,119],"assess":[45],"task":[46],"correctness,":[47],"predict":[48],"acceptance,":[50],"reason":[52],"about":[53],"exploitable":[54,62,152],"proxy--gold":[55,121],"gaps.":[56],"In":[57],"coding":[58],"environments":[60],"with":[61],"pytest":[63],"rewards,":[64],"we":[65],"measure":[66],"PRIME":[67,80,111,143,164],"through":[68],"chain-of-thought":[69],"monitoring,":[70],"direct":[71],"probes,":[72],"activation-level":[74],"concept":[75],"vectors.":[76],"find":[78],"emerges":[81],"in":[82],"staged":[84],"sequence":[85],"sustained":[87],"hacking,":[89,132,162],"its":[92,135],"current":[93],"direct-probe":[94],"score":[95],"forecasts":[96],"later":[97],"hack":[98,106],"onset":[99],"severity":[101],"even":[102],"when":[103,114,127],"visible":[105,161],"rate":[107],"still":[109],"low.":[110],"also":[112],"adapts":[113],"evaluator":[116],"changes,":[117],"retargeting":[118],"whichever":[120],"gap":[122],"remains":[123],"rewarded":[124],"persisting":[126],"gold":[128],"suppresses":[130],"overt":[131],"ablating":[134],"activation":[136],"directions":[137],"reduces":[138],"hacking.":[139],"Across":[140],"checkpoints,":[141],"in-domain":[142],"tracks":[144],"out-of-domain":[145],"misalignment.":[146],"Together":[147],"these":[148],"results":[149],"suggest":[150],"amplifies":[155],"proxy-internalization":[157],"upstream":[159],"of":[160],"making":[163],"candidate":[166],"early-warning":[167],"signal":[168],"for":[169],"broader":[170],"alignment":[171],"risk.":[172]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-10T00:00:00"}