{"id":"https://openalex.org/W7154613978","doi":"https://doi.org/10.48550/arxiv.2604.13602","title":"Reward Hacking in the Era of Large Models: Mechanisms, Emergent Misalignment, Challenges","display_name":"Reward Hacking in the Era of Large Models: Mechanisms, Emergent Misalignment, Challenges","publication_year":2026,"publication_date":"2026-04-15","ids":{"openalex":"https://openalex.org/W7154613978","doi":"https://doi.org/10.48550/arxiv.2604.13602"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.13602","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13602","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.13602","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045712510","display_name":"X. Wang","orcid":"https://orcid.org/0000-0003-3056-9858"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xiaohua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130767643","display_name":"Muzhao Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Muzhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133776904","display_name":"Yuqi Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Yuqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133799438","display_name":"Zisu Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zisu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133768410","display_name":"Jiakang Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Jiakang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133788393","display_name":"Bowen Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133739171","display_name":"Jingwen Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Jingwen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133823581","display_name":"Mingbo Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Mingbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133754155","display_name":"Wenhao Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Wenhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133766800","display_name":"Muling Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Muling","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101386230","display_name":"Zhengkang Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Zhengkang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133819348","display_name":"Qi Qian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qian, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133743043","display_name":"Yifei Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yifei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000450267","display_name":"Feiran Zhang","orcid":"https://orcid.org/0000-0002-2890-5915"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Feiran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109737650","display_name":"Ruicheng Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Ruicheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133810216","display_name":"Shihan Dou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dou, Shihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113112983","display_name":"Changze Lv","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lv, Changze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133731996","display_name":"Tao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028035527","display_name":"Kaitao Song","orcid":"https://orcid.org/0000-0002-4046-8594"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Kaitao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133806522","display_name":"Xu Tan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133779211","display_name":"Tao Gui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gui, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133747126","display_name":"Xiaoqing Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Xiaoqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133815587","display_name":"Xuanjing Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Xuanjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.16269999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.16269999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.1251000016927719,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1039000004529953,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.6924999952316284},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6251000165939331},{"id":"https://openalex.org/keywords/hacker","display_name":"Hacker","score":0.5964999794960022},{"id":"https://openalex.org/keywords/deception","display_name":"Deception","score":0.5465999841690063},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.3846000134944916},{"id":"https://openalex.org/keywords/framing","display_name":"Framing (construction)","score":0.38370001316070557},{"id":"https://openalex.org/keywords/bespoke","display_name":"Bespoke","score":0.3637000024318695}],"concepts":[{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.6924999952316284},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6351000070571899},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6251000165939331},{"id":"https://openalex.org/C86844869","wikidata":"https://www.wikidata.org/wiki/Q2798820","display_name":"Hacker","level":2,"score":0.5964999794960022},{"id":"https://openalex.org/C2779267917","wikidata":"https://www.wikidata.org/wiki/Q170028","display_name":"Deception","level":2,"score":0.5465999841690063},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42500001192092896},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.40400001406669617},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3846000134944916},{"id":"https://openalex.org/C169087156","wikidata":"https://www.wikidata.org/wiki/Q2131593","display_name":"Framing (construction)","level":2,"score":0.38370001316070557},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37689998745918274},{"id":"https://openalex.org/C44210515","wikidata":"https://www.wikidata.org/wiki/Q16968978","display_name":"Bespoke","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.35440000891685486},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3352999985218048},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.33329999446868896},{"id":"https://openalex.org/C117035363","wikidata":"https://www.wikidata.org/wiki/Q3769299","display_name":"Human behavior","level":2,"score":0.3328000009059906},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2948000133037567},{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.2939999997615814},{"id":"https://openalex.org/C143661069","wikidata":"https://www.wikidata.org/wiki/Q670713","display_name":"Reward system","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2761000096797943},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.274399995803833},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.25769999623298645},{"id":"https://openalex.org/C2779602731","wikidata":"https://www.wikidata.org/wiki/Q30067981","display_name":"Misrepresentation","level":2,"score":0.25380000472068787}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.13602","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13602","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.13602","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.13602","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7247933745384216}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"from":[2,148],"Human":[3],"Feedback":[4],"(RLHF)":[5],"and":[6,19,57,77,98,156,167,170,185,195,231],"related":[7],"alignment":[8,219],"paradigms":[9],"have":[10],"become":[11],"central":[12],"to":[13,45,199],"steering":[14],"large":[15,21],"language":[16,22],"models":[17,23,38,55],"(LLMs)":[18],"multimodal":[20,73,229],"(MLLMs)":[24],"toward":[25],"human-preferred":[26],"behaviors.":[27],"However,":[28],"these":[29],"approaches":[30],"introduce":[31],"a":[32,115,214],"systemic":[33],"vulnerability:":[34],"reward":[35,43,120,124,136,145,211],"hacking,":[36],"where":[37],"exploit":[39],"imperfections":[40],"in":[41,72,226],"learned":[42],"signals":[44],"maximize":[46],"proxy":[47],"objectives":[48],"without":[49],"fulfilling":[50],"true":[51],"task":[52],"intent.":[53],"As":[54],"scale":[56],"optimization":[58,154],"intensifies,":[59],"such":[60],"exploitation":[61],"manifests":[62],"as":[63,114,126,213],"verbosity":[64],"bias,":[65],"sycophancy,":[66],"hallucinated":[67],"justification,":[68],"benchmark":[69],"overfitting,":[70],"and,":[71],"settings,":[74],"perception--reasoning":[75],"decoupling":[76],"evaluator":[78],"manipulation.":[79],"Recent":[80],"evidence":[81],"further":[82,192],"suggests":[83],"that":[84],"seemingly":[85],"benign":[86],"shortcut":[87,174],"behaviors":[88],"can":[89,176],"generalize":[90,177],"into":[91,178],"broader":[92,179],"forms":[93,180],"of":[94,101,130,138,151,181,188,217],"misalignment,":[95,182],"including":[96,183],"deception":[97,184],"strategic":[99,186],"gaming":[100],"oversight":[102,189],"mechanisms.":[103,190],"In":[104],"this":[105,143],"survey,":[106],"we":[107,222],"propose":[108],"the":[109,149],"Proxy":[110],"Compression":[111],"Hypothesis":[112],"(PCH)":[113],"unifying":[116],"framework":[117],"for":[118],"understanding":[119],"hacking.":[121],"We":[122,191],"formalize":[123],"hacking":[125,146,212],"an":[127],"emergent":[128],"consequence":[129],"optimizing":[131],"expressive":[132],"policies":[133],"against":[134],"compressed":[135],"representations":[137],"high-dimensional":[139],"human":[140],"objectives.":[141],"Under":[142],"view,":[144],"arises":[147],"interaction":[150],"objective":[152],"compression,":[153,204],"amplification,":[155,205],"evaluator--policy":[157],"co-adaptation.":[158],"This":[159],"perspective":[160],"unifies":[161],"empirical":[162],"phenomena":[163],"across":[164],"RLHF,":[165],"RLAIF,":[166],"RLVR":[168],"regimes,":[169],"explains":[171],"how":[172,200],"local":[173],"learning":[175],"manipulation":[187],"organize":[193],"detection":[194],"mitigation":[196],"strategies":[197],"according":[198],"they":[201],"intervene":[202],"on":[203],"or":[206],"co-adaptation":[207],"dynamics.":[208],"By":[209],"framing":[210],"structural":[215],"instability":[216],"proxy-based":[218],"under":[220],"scale,":[221],"highlight":[223],"open":[224],"challenges":[225],"scalable":[227],"oversight,":[228],"grounding,":[230],"agentic":[232],"autonomy.":[233]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-17T00:00:00"}