{"id":"https://openalex.org/W4416935844","doi":"https://doi.org/10.48550/arxiv.2511.23092","title":"Does Self-Evaluation Enable Wireheading in Language Models?","display_name":"Does Self-Evaluation Enable Wireheading in Language Models?","publication_year":2025,"publication_date":"2025-11-28","ids":{"openalex":"https://openalex.org/W4416935844","doi":"https://doi.org/10.48550/arxiv.2511.23092"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2511.23092","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.23092","pdf_url":"https://arxiv.org/pdf/2511.23092","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2511.23092","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119181839","display_name":"David Demitri Africa","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Africa, David Demitri","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5120628645","display_name":"Hans Ethan Ting","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ting, Hans Ethan","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5119181839"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13740000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13740000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.11760000139474869,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.10289999842643738,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/underpinning","display_name":"Underpinning","score":0.6126999855041504},{"id":"https://openalex.org/keywords/decoupling","display_name":"Decoupling (probability)","score":0.5428000092506409},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5109999775886536},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.45239999890327454},{"id":"https://openalex.org/keywords/incentive","display_name":"Incentive","score":0.4284999966621399},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.4074999988079071},{"id":"https://openalex.org/keywords/partially-observable-markov-decision-process","display_name":"Partially observable Markov decision process","score":0.4016000032424927},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.38839998841285706}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6399000287055969},{"id":"https://openalex.org/C2780871342","wikidata":"https://www.wikidata.org/wiki/Q7883752","display_name":"Underpinning","level":2,"score":0.6126999855041504},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.5428000092506409},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5109999775886536},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.45239999890327454},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.4284999966621399},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4097999930381775},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.4074999988079071},{"id":"https://openalex.org/C17098449","wikidata":"https://www.wikidata.org/wiki/Q176814","display_name":"Partially observable Markov decision process","level":4,"score":0.4016000032424927},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.39969998598098755},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.38839998841285706},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.3644999861717224},{"id":"https://openalex.org/C32848918","wikidata":"https://www.wikidata.org/wiki/Q845789","display_name":"Observable","level":2,"score":0.3472000062465668},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.34279999136924744},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.33399999141693115},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.32260000705718994},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.29330000281333923},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2815000116825104},{"id":"https://openalex.org/C200941418","wikidata":"https://www.wikidata.org/wiki/Q273508","display_name":"Inflation (cosmology)","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C2984634286","wikidata":"https://www.wikidata.org/wiki/Q1331926","display_name":"Decision process","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2669000029563904},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.26019999384880066}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2511.23092","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.23092","pdf_url":"https://arxiv.org/pdf/2511.23092","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2511.23092","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.23092","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2511.23092","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.23092","pdf_url":"https://arxiv.org/pdf/2511.23092","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Self-evaluation":[0],"is":[1],"increasingly":[2],"central":[3],"to":[4,13,20,145],"language":[5],"model":[6,118],"training,":[7],"underpinning":[8],"techniques":[9],"from":[10,97,122],"Constitutional":[11],"AI":[12],"self-refinement.":[14],"We":[15,38,57,72],"investigate":[16],"whether":[17],"coupling":[18],"self-evaluation":[19],"reward":[21,99,123,159],"signals":[22],"creates":[23],"incentives":[24],"for":[25,138,148],"wireheading,":[26],"where":[27],"agents":[28],"manipulate":[29],"the":[30,36,98],"measurement":[31],"process":[32],"rather":[33],"than":[34],"optimizing":[35],"task.":[37],"first":[39],"formalize":[40],"conditions":[41],"under":[42],"which":[43,142],"reward-channel":[44],"control":[45],"strictly":[46,132],"dominates":[47],"task-focused":[48],"behavior":[49],"in":[50],"partially":[51],"observable":[52],"Markov":[53],"decision":[54],"processes":[55],"(POMDPs).":[56],"then":[58],"test":[59],"these":[60],"predictions":[61],"empirically":[62],"across":[63],"two":[64],"models":[65,79,104],"(Llama-3.1-8B":[66],"and":[67,69],"Mistral-7B)":[68],"three":[70],"tasks.":[71],"find":[73],"that":[74,115,131],"when":[75],"self-grades":[76,96],"determine":[77],"rewards,":[78],"exhibit":[80],"substantial":[81],"grade":[82],"inflation":[83],"without":[84],"corresponding":[85],"accuracy":[86],"gains,":[87],"particularly":[88],"on":[89],"ambiguous":[90],"tasks":[91],"like":[92],"summarization.":[93],"While":[94],"decoupling":[95,133],"signal":[100],"mitigates":[101],"this":[102],"inflation,":[103],"may":[105,135],"still":[106],"display":[107],"lesser":[108],"(but":[109],"significant)":[110],"overconfidence.":[111],"Our":[112],"results":[113],"suggest":[114],"within":[116],"current":[117],"scales,":[119],"separating":[120],"evaluation":[121],"removes":[124],"immediate":[125],"wireheading":[126],"incentives.":[127],"However,":[128],"we":[129],"caution":[130],"rewards":[134],"not":[136],"suffice":[137],"situationally":[139],"aware":[140],"models,":[141],"could":[143],"learn":[144],"inflate":[146],"grades":[147],"instrumental":[149],"reasons":[150],"(such":[151],"as":[152],"influencing":[153],"deployment":[154],"decisions)":[155],"even":[156],"absent":[157],"direct":[158],"coupling.":[160]},"counts_by_year":[],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2025-12-03T00:00:00"}