{"id":"https://openalex.org/W7160849758","doi":"https://doi.org/10.48550/arxiv.2605.07331","title":"Rethinking Importance Sampling in LLM Policy Optimization: A Cumulative Token Perspective","display_name":"Rethinking Importance Sampling in LLM Policy Optimization: A Cumulative Token Perspective","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160849758","doi":"https://doi.org/10.48550/arxiv.2605.07331"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.07331","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07331","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.07331","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135907190","display_name":"Yuheng Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072672530","display_name":"Chenlu Ye","orcid":"https://orcid.org/0000-0002-0243-4449"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Chenlu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135296145","display_name":"Shuowei Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Shuowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135828261","display_name":"Changlong Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Changlong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135854957","display_name":"Wei Xiong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiong, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135866006","display_name":"Saurabh Sahu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sahu, Saurabh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135887960","display_name":"Nan Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Nan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6176999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6176999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.051100000739097595,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.0272000003606081,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7175999879837036},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5425000190734863},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.41670000553131104},{"id":"https://openalex.org/keywords/multiplicative-function","display_name":"Multiplicative function","score":0.40709999203681946},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.3880000114440918},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.3792000114917755},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.36980000138282776},{"id":"https://openalex.org/keywords/importance-sampling","display_name":"Importance sampling","score":0.36059999465942383}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7175999879837036},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5425000190734863},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5149999856948853},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.41670000553131104},{"id":"https://openalex.org/C42747912","wikidata":"https://www.wikidata.org/wiki/Q1048447","display_name":"Multiplicative function","level":2,"score":0.40709999203681946},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.3880000114440918},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3792000114917755},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.36980000138282776},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.36059999465942383},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.3544999957084656},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.34040001034736633},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.33489999175071716},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.31949999928474426},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3172000050544739},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3127000033855438},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.3125},{"id":"https://openalex.org/C103784038","wikidata":"https://www.wikidata.org/wiki/Q386228","display_name":"Cumulative distribution function","level":3,"score":0.3073999881744385},{"id":"https://openalex.org/C136625980","wikidata":"https://www.wikidata.org/wiki/Q663208","display_name":"Rounding","level":2,"score":0.2919999957084656},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.27649998664855957},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27549999952316284},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2522999942302704}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.07331","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07331","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.07331","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07331","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning,":[1],"including":[2],"reinforcement":[3],"learning":[4],"with":[5,186,238],"verifiable":[6],"rewards":[7],"(RLVR),":[8],"has":[9,159],"emerged":[10],"as":[11,46,130],"a":[12,39,131],"powerful":[13],"approach":[14],"for":[15,153],"LLM":[16],"post-training.":[17],"Central":[18],"to":[19,81,127,135,195],"these":[20],"approaches":[21],"is":[22],"the":[23,26,82,101,106,116,121,142,164,181,196,201,218,229],"design":[24],"of":[25,85,103,123,200],"importance":[27],"sampling":[28],"(IS)":[29],"ratio":[30,147,185],"used":[31],"in":[32,217],"off-policy":[33],"policy-gradient":[34,144],"estimation.":[35],"Existing":[36],"methods":[37],"face":[38],"fundamental":[40],"bias-variance":[41],"dilemma:":[42],"token-level":[43,143,155],"IS":[44,109,119,184],"ratios,":[45,87],"adopted":[47],"by":[48,62],"PPO":[49],"(Schulman":[50],"et":[51,57,91],"al.,":[52,58,92],"2017)":[53],"and":[54,158,214,241],"GRPO":[55,240],"(Shao":[56],"2024),":[59],"introduce":[60],"bias":[61],"ignoring":[63],"prefix":[64,151],"state":[65],"distribution":[66],"mismatch;":[67],"full":[68,165],"sequence":[69,166],"ratios":[70,125],"provide":[71],"exact":[72,107],"trajectory-level":[73],"correction":[74,152],"but":[75],"suffer":[76],"from":[77,105],"high":[78],"variance":[79,162],"due":[80],"multiplicative":[83],"accumulation":[84],"per-token":[86,124],"while":[88],"GSPO":[89,242],"(Zheng":[90],"2025)":[93],"improves":[94],"numerical":[95],"stability":[96],"via":[97],"length":[98],"normalization":[99],"at":[100,248],"cost":[102],"deviating":[104],"full-sequence":[108],"correction.":[110],"In":[111],"this":[112,136,146,170],"work,":[113],"we":[114,172],"identify":[115],"cumulative":[117,182,202],"token":[118,183,210],"ratio,":[120],"product":[122],"up":[126],"position":[128],"$t$,":[129],"theoretically":[132],"principled":[133],"solution":[134],"dilemma.":[137],"We":[138,212],"prove":[139],"that,":[140],"under":[141],"formulation,":[145],"provides":[148],"an":[149],"unbiased":[150],"each":[154],"gradient":[156],"term":[157],"strictly":[160],"lower":[161],"than":[163],"ratio.":[167],"Building":[168],"on":[169,222],"insight,":[171],"propose":[173],"CTPO":[174,216],"(Cumulative":[175],"Token":[176],"Policy":[177],"Optimization),":[178],"which":[179],"combines":[180],"position-adaptive":[187],"clipping":[188],"that":[189],"scales":[190,236],"log-space":[191],"clip":[192],"bounds":[193],"according":[194],"natural":[197],"$\\sqrt{t}$":[198],"growth":[199],"log-ratio.":[203],"This":[204],"yields":[205],"more":[206],"consistent":[207],"regularization":[208],"across":[209,233],"positions.":[211],"implement":[213],"evaluate":[215],"tool-integrated":[219],"reasoning":[220,226],"setting":[221],"several":[223],"challenging":[224],"mathematical":[225],"benchmarks,":[227],"achieving":[228],"best":[230],"average":[231],"performance":[232],"both":[234],"model":[235],"compared":[237],"strong":[239],"baselines.":[243],"Code":[244],"will":[245],"be":[246],"available":[247],"https://github.com/horizon-llm/CTPO.":[249]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-12T00:00:00"}