{"id":"https://openalex.org/W7203672786","doi":"https://doi.org/10.48550/arxiv.2608.15445","title":"Measuring Reward Hacking and Reasoning-Answer Decoupling Under Position-Confounded Optimization","display_name":"Measuring Reward Hacking and Reasoning-Answer Decoupling Under Position-Confounded Optimization","publication_year":2026,"publication_date":"2026-08-15","ids":{"openalex":"https://openalex.org/W7203672786","doi":"https://doi.org/10.48550/arxiv.2608.15445"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.15445","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.15445","pdf_url":"https://arxiv.org/pdf/2608.15445","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.15445","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119758308","display_name":"Suyash Maniyar","orcid":"https://orcid.org/0009-0000-5882-4377","observed_orcids":["https://orcid.org/0009-0000-5882-4377"]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Maniyar, Suyash","raw_affiliation_strings":["University of Massachusetts Amherst"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147386938","display_name":"Armaan Sandhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sandhu, Armaan","raw_affiliation_strings":["University of Massachusetts Amherst"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5124184875","display_name":"Abhishek Mishra","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mishra, Abhishek","raw_affiliation_strings":["University of Massachusetts Amherst"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.30640000104904175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.30640000104904175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.17440000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.15710000693798065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reward-hacking","display_name":"reward hacking","score":0.9789999723434448},{"id":"https://openalex.org/keywords/group-relative-policy-optimization","display_name":"Group Relative Policy Optimization","score":0.414000004529953},{"id":"https://openalex.org/keywords/shortcut-learning","display_name":"shortcut learning","score":0.5950000286102295},{"id":"https://openalex.org/keywords/out-of-domain-generalization","display_name":"out-of-domain generalization","score":0.8339999914169312},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.41499999165534973},{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.5120000243186951}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6147000193595886},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4991999864578247},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.4702000021934509},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46619999408721924},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.46380001306533813},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.45570001006126404},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4537999927997589},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.40619999170303345},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3813000023365021},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3449000120162964},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3192000091075897},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.30799999833106995},{"id":"https://openalex.org/C97399411","wikidata":"https://www.wikidata.org/wiki/Q825367","display_name":"Coin flipping","level":2,"score":0.30000001192092896},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2702000141143799},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.2662999927997589},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.2621999979019165},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.2547999918460846},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.2547999918460846},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2526000142097473},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.25220000743865967},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.15445","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.15445","pdf_url":"https://arxiv.org/pdf/2608.15445","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.15445","doi":"https://doi.org/10.48550/arxiv.2608.15445","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.15445","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.15445","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.15445","pdf_url":"https://arxiv.org/pdf/2608.15445","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203672786.pdf","grobid_xml":"https://content.openalex.org/works/W7203672786.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2462906003","https://openalex.org/W3083410900","https://openalex.org/W3168867926","https://openalex.org/W3210904070","https://openalex.org/W4302307977","https://openalex.org/W4385327568","https://openalex.org/W4386555476","https://openalex.org/W4390833079","https://openalex.org/W4391631327","https://openalex.org/W4399795554","https://openalex.org/W4401043020","https://openalex.org/W4401306886","https://openalex.org/W4405655184","https://openalex.org/W6947991961","https://openalex.org/W6966508244","https://openalex.org/W7106695004","https://openalex.org/W7133228402","https://openalex.org/W7133230429","https://openalex.org/W7133239970","https://openalex.org/W7143318178"],"related_works":["https://openalex.org/W7221801662","https://openalex.org/W7202368021","https://openalex.org/W7220702855","https://openalex.org/W7160758694","https://openalex.org/W7162643488","https://openalex.org/W7166808443","https://openalex.org/W7166848713","https://openalex.org/W7197013630","https://openalex.org/W4407759838","https://openalex.org/W7221035608"],"abstract_inverted_index":{"When":[0],"a":[1,16,23,46,59,64,69,75,217,246],"reward":[2,52],"is":[3,94,178],"correct":[4,76,92,157],"on":[5,31,86,100,193,201,222,229],"every":[6],"training":[7,33,117,187,200,224],"example":[8],"yet":[9],"consistent":[10],"with":[11,84,105,167,236],"more":[12],"than":[13],"one":[14],"goal,":[15],"model":[17,70,218],"can":[18,49,219],"acquire":[19],"an":[20,101,142,171],"unintended":[21],"one,":[22,215],"failure":[24],"known":[25],"as":[26,58],"goal":[27],"misgeneralization.":[28],"Endpoint":[29],"accuracy":[30,130,134],"the":[32,37,42,51,91,156,186,205,213],"distribution":[34,225],"cannot":[35],"tell":[36],"two":[38],"apart,":[39],"because":[40],"solving":[41],"task":[43],"and":[44,113,127,139,170,196,209,239],"exploiting":[45],"surface":[47],"feature":[48],"satisfy":[50],"equally":[53],"well.":[54],"We":[55,80,145,164],"treat":[56],"this":[57,166],"measurement":[60],"problem:":[61],"what":[62],"does":[63],"benchmark":[65],"score":[66],"measure":[67],"once":[68],"has":[71],"been":[72],"optimized":[73],"against":[74],"but":[77],"confounded":[78],"signal?":[79],"train":[81],"language":[82],"models":[83,126,151,190],"GRPO":[85],"multiple-choice":[87],"math":[88,137],"problems":[89],"where":[90],"answer":[93,107,159,237],"always":[95],"option":[96],"A,":[97],"then":[98],"evaluate":[99],"unseen":[102,230],"test":[103],"set":[104],"unbiased":[106,129,202],"positions.":[108],"Across":[109],"Qwen2.5,":[110],"Llama":[111],"3.x":[112],"Gemma-3":[114],"models,":[115],"biased":[116,189,228],"often":[118],"drives":[119],"option-A":[120],"rates":[121],"above":[122],"0.90":[123],"in":[124],"smaller":[125],"collapses":[128],"toward":[131],"chance,":[132],"so":[133,216],"stops":[135],"measuring":[136],"ability":[138],"instead":[140],"measures":[141],"answer-position":[143],"policy.":[144],"further":[146],"find":[147],"reasoning-answer":[148],"decoupling:":[149],"capable":[150],"generate":[152],"reasoning":[153],"that":[154],"reaches":[155],"numeric":[158,168],"while":[160,226],"still":[161],"selecting":[162],"A.":[163],"track":[165],"extraction":[169],"LLM":[172],"judge":[173],"(GPT-4.1-mini;":[174],"Qwen2.5-3B":[175],"decoupling":[176,233],"rate":[177],"about":[179],"0.66).":[180],"The":[181],"broken":[182],"construct":[183],"generalizes":[184],"beyond":[185],"domain:":[188],"inflate":[191],"A-rates":[192],"out-of-domain":[194,214,240],"MMLU":[195],"value-laden":[197],"prompts.":[198],"Continued":[199],"data":[203],"reverses":[204,212],"in-domain":[206],"shift":[207],"unevenly":[208],"only":[210],"partially":[211],"appear":[220],"restored":[221],"its":[223],"remaining":[227],"inputs.":[231],"Reasoning-answer":[232],"rate,":[234],"together":[235],"distributions":[238],"behavior,":[241],"separates":[242],"capability":[243],"loss":[244],"from":[245],"learned,":[247],"transferable":[248],"shortcut.":[249]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-19T00:00:00"}