{"id":"https://openalex.org/W7160896030","doi":"https://doi.org/10.48550/arxiv.2605.08666","title":"The Cancellation Hypothesis in Critic-Free RL: From Outcome Rewards to Token Credits","display_name":"The Cancellation Hypothesis in Critic-Free RL: From Outcome Rewards to Token Credits","publication_year":2026,"publication_date":"2026-05-09","ids":{"openalex":"https://openalex.org/W7160896030","doi":"https://doi.org/10.48550/arxiv.2605.08666"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.08666","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08666","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.08666","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067076835","display_name":"Tianhao Cheng","orcid":"https://orcid.org/0000-0002-7737-4639"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Tianhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135934925","display_name":"Zeyu Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zeyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135993662","display_name":"Zihan Qiu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Zihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135969744","display_name":"Yu Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135966912","display_name":"Edoardo Ponti","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ponti, Edoardo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135931945","display_name":"Yinghui Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Yinghui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135998770","display_name":"Ivan Titov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Titov, Ivan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135989170","display_name":"Zenglin Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Zenglin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.20419999957084656,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.20419999957084656,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.07959999889135361,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.05860000103712082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7498999834060669},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.5666000247001648},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.43309998512268066},{"id":"https://openalex.org/keywords/psychological-intervention","display_name":"Psychological intervention","score":0.3391999900341034},{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.328000009059906},{"id":"https://openalex.org/keywords/empirical-evidence","display_name":"Empirical evidence","score":0.2809999883174896}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7498999834060669},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.652400016784668},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.5666000247001648},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.43309998512268066},{"id":"https://openalex.org/C27415008","wikidata":"https://www.wikidata.org/wiki/Q7256382","display_name":"Psychological intervention","level":2,"score":0.3391999900341034},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.328000009059906},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32499998807907104},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.2809999883174896},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2768999934196472},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.27410000562667847},{"id":"https://openalex.org/C127729010","wikidata":"https://www.wikidata.org/wiki/Q60165","display_name":"Dynamic inconsistency","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C155076860","wikidata":"https://www.wikidata.org/wiki/Q1932004","display_name":"Token economy","level":3,"score":0.2583000063896179},{"id":"https://openalex.org/C131584629","wikidata":"https://www.wikidata.org/wiki/Q4308705","display_name":"Coupling (piping)","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.08666","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08666","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.08666","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08666","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"A":[0],"commonly":[1],"accepted":[2],"explanation":[3],"of":[4,49,121,196],"critic-free":[5,31,170,186,220,253],"RL":[6,32,58,171,187,221,254],"for":[7,127,252],"LLMs,":[8],"based":[9],"on":[10,166],"sequence-level":[11],"rewards,":[12],"is":[13,73],"that":[14,67,102],"it":[15],"reinforces":[16],"successful":[17,140],"rollouts":[18,44,141],"with":[19,84,106,158,164],"a":[20,34,68,89,119,248],"positive":[21,41,131,168,201],"advantage":[22],"while":[23,135],"penalizing":[24],"failed":[25],"ones.":[26],"In":[27],"contrast,":[28],"we":[29,64,113,209],"study":[30],"from":[33,151,174,200],"token-level":[35,148],"perspective,":[36],"revealing":[37],"the":[38,115],"token-flipping":[39],"phenomenon:":[40],"and":[42,132,176,225,247],"negative":[43,133,203],"exhibit":[45],"remarkably":[46],"similar":[47],"proportions":[48],"tokens":[50,86,101,128,136,178],"whose":[51],"probabilities":[52],"are":[53,103],"boosted":[54,184],"or":[55,202,216],"suppressed":[56,193],"during":[57],"training.":[59,255],"To":[60],"explain":[61],"this":[62,111,156,207],"phenomenon,":[63],"further":[65],"show":[66],"token's":[69],"change":[70],"in":[71,219],"probability":[72],"not":[74],"fully":[75],"determined":[76],"by":[77,130,185,206],"its":[78],"own":[79],"advantage;":[80],"coupled":[81],"gradient":[82],"interactions":[83],"other":[85],"also":[87],"play":[88],"non-negligible":[90],"role.":[91],"Specifically,":[92],"these":[93,231],"token":[94],"coupling":[95],"effects":[96],"occur":[97],"primarily":[98],"between":[99],"identical":[100],"both":[104,243],"predicted":[105],"low":[107],"confidence.":[108],"Building":[109],"upon":[110],"analysis,":[112],"propose":[114],"cancellation":[116,218,241],"hypothesis:":[117],"as":[118,242],"result":[120],"coupling,":[122],"opposing":[123],"signals":[124],"cancel":[125],"out":[126],"shared":[129],"rollouts,":[134,169],"more":[137],"specific":[138],"to":[139,214],"receive":[142],"stronger":[143],"reinforcement,":[144],"thereby":[145],"inducing":[146],"hidden":[147],"credit":[149],"assignment":[150],"rollout-level":[152],"rewards.":[153],"We":[154],"support":[155],"hypothesis":[157],"complementary":[159],"empirical":[160],"evidence.":[161],"(1)":[162],"Compared":[163],"training":[165,235],"only":[167],"shifts":[172],"updates":[173],"template":[175],"formatting":[177],"toward":[179],"reasoning":[180],"tokens;":[181],"(2)":[182],"Tokens":[183],"consistently":[188],"demonstrate":[189],"higher":[190],"value":[191],"than":[192],"tokens,":[194],"regardless":[195],"whether":[197],"they":[198],"originate":[199],"rollouts.":[204],"Guided":[205],"view,":[208],"implement":[210],"two":[211],"batching":[212],"interventions":[213,232],"encourage":[215],"preserve":[217],"training:":[222],"query-preserved":[223],"mini-batching":[224],"reward-balanced":[226],"batching.":[227],"Despite":[228],"their":[229],"simplicity,":[230],"improve":[233],"RLVR":[234],"across":[236],"multiple":[237],"model":[238],"scales,":[239],"supporting":[240],"an":[244],"explanatory":[245],"principle":[246],"practical":[249],"design":[250],"criterion":[251]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-13T00:00:00"}