{"id":"https://openalex.org/W7162112465","doi":"https://doi.org/10.48550/arxiv.2605.21851","title":"OPPO: Bayesian Value Recursion for Token-Level Credit Assignment in LLM Reasoning","display_name":"OPPO: Bayesian Value Recursion for Token-Level Credit Assignment in LLM Reasoning","publication_year":2026,"publication_date":"2026-05-21","ids":{"openalex":"https://openalex.org/W7162112465","doi":"https://doi.org/10.48550/arxiv.2605.21851"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.21851","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.21851","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.21851","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136777685","display_name":"Yu Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136769917","display_name":"Rui Miao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miao, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136764247","display_name":"Tian Lan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lan, Tian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136785439","display_name":"Zhengling Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Zhengling","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5001999735832214,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5001999735832214,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.10400000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.04780000075697899,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5673999786376953},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.5353000164031982},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.5230000019073486},{"id":"https://openalex.org/keywords/recursion","display_name":"Recursion (computer science)","score":0.43860000371932983},{"id":"https://openalex.org/keywords/signal","display_name":"SIGNAL (programming language)","score":0.4251999855041504},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.41659998893737793},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.4000999927520752},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.38760000467300415},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3686000108718872}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6085000038146973},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5673999786376953},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.5353000164031982},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.5230000019073486},{"id":"https://openalex.org/C168773036","wikidata":"https://www.wikidata.org/wiki/Q264164","display_name":"Recursion (computer science)","level":2,"score":0.43860000371932983},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.4251999855041504},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.41659998893737793},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.4000999927520752},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3939000070095062},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3887999951839447},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.38760000467300415},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.382999986410141},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3686000108718872},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.3628999888896942},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.351500004529953},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.336899995803833},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.33059999346733093},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C6177178","wikidata":"https://www.wikidata.org/wiki/Q10998070","display_name":"Discounting","level":2,"score":0.28439998626708984},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.28279998898506165},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C123197309","wikidata":"https://www.wikidata.org/wiki/Q2882343","display_name":"Multi-armed bandit","level":3,"score":0.2761000096797943},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.274399995803833},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2540999948978424},{"id":"https://openalex.org/C33724603","wikidata":"https://www.wikidata.org/wiki/Q812540","display_name":"Bayesian network","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.21851","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.21851","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.21851","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.21851","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.5568562746047974,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"with":[2,134,175,252,257],"verifiable":[3],"rewards":[4],"has":[5],"become":[6],"the":[7,15,28,60,80,93,98,105,116,127,152,155,191,197,201],"standard":[8],"recipe":[9],"for":[10,88],"improving":[11],"LLM":[12],"reasoning,":[13],"but":[14],"dominant":[16],"algorithm":[17],"GRPO":[18],"assigns":[19],"a":[20,77,108,123,135,165,176,193,206,211,217],"single":[21,78],"trajectory-level":[22,61],"advantage":[23,137,153],"to":[24,65,216,242],"every":[25,131],"token,":[26],"diluting":[27],"signal":[29,56,82,106,158],"at":[30,37,115,130],"pivotal":[31,173],"reasoning":[32,231],"steps":[33],"and":[34,114,144,199,210,229,238,247],"injecting":[35],"noise":[36],"uninformative":[38],"ones.":[39],"Critic-free":[40],"alternatives":[41],"derived":[42],"from":[43,59],"on-policy":[44,202],"distillation":[45,161,203],"supply":[46],"per-token":[47,156],"signals":[48],"through":[49],"oracle-conditioned":[50],"likelihood":[51],"ratios,":[52],"yet":[53],"apply":[54],"each":[55],"in":[57,111,187],"isolation":[58],"evidence":[62],"accumulated":[63],"up":[64,241],"that":[66,138,168,195,213,254],"position.":[67],"We":[68],"propose":[69],"Oracle-Prompted":[70],"Policy":[71],"Optimization":[72],"(OPPO),":[73],"which":[74,188],"rests":[75],"on":[76,171,245,250],"observation:":[79],"oracle":[81],"used":[83,159],"by":[84,160,164,240],"prior":[85],"distillation-style":[86],"methods":[87,162],"local":[89],"discrimination":[90,157],"is":[91],"also":[92],"natural":[94],"Bayesian":[95],"update":[96],"of":[97,118,126],"model's":[99],"belief":[100],"about":[101],"eventual":[102],"success.":[103],"Accumulating":[104],"along":[107],"trajectory":[109],"yields,":[110],"closed":[112],"form":[113],"cost":[117],"one":[119],"extra":[120],"forward":[121],"pass,":[122],"running":[124],"estimate":[125],"success":[128],"probability":[129],"position,":[132],"together":[133],"token-level":[136],"requires":[139],"no":[140,145],"learned":[141],"value":[142],"network":[143],"additional":[146],"rollouts.":[147],"A":[148],"first-order":[149],"analysis":[150],"factorizes":[151],"into":[154],"modulated":[163],"state":[166],"weight":[167],"concentrates":[169],"credit":[170],"genuinely":[172],"tokens,":[174],"directional":[177],"variance-reduction":[178],"guarantee.":[179],"The":[180],"framework":[181],"admits":[182],"two":[183,222],"estimators":[184],"differing":[185],"only":[186],"model":[189],"scores":[190],"evidence:":[192],"\\textit{self-oracle}":[194],"reuses":[196],"student":[198],"recovers":[200],"reward":[204],"as":[205],"strict":[207],"special":[208],"case,":[209],"\\textit{teacher-oracle}":[212],"delegates":[214],"scoring":[215],"stronger":[218],"frozen":[219],"model.":[220],"On":[221],"base":[223],"LLMs":[224],"across":[225],"seven":[226],"mathematics,":[227],"science,":[228],"code":[230],"benchmarks,":[232],"OPPO":[233],"improves":[234],"over":[235],"GRPO,":[236],"DAPO,":[237],"SDPO":[239],"$+6.0$":[243],"points":[244,249],"AMC'23":[246],"$+5.2$":[248],"AIME'24,":[251],"gains":[253],"widen":[255],"monotonically":[256],"response":[258],"length.":[259]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-23T00:00:00"}