{"id":"https://openalex.org/W4415317764","doi":"https://doi.org/10.48550/arxiv.2509.13232","title":"Single-stream Policy Optimization","display_name":"Single-stream Policy Optimization","publication_year":2025,"publication_date":"2025-09-16","ids":{"openalex":"https://openalex.org/W4415317764","doi":"https://doi.org/10.48550/arxiv.2509.13232"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2509.13232","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.13232","pdf_url":"https://arxiv.org/pdf/2509.13232","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2509.13232","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062979221","display_name":"Zhongwen Xu","orcid":"https://orcid.org/0000-0002-9348-7262","observed_orcids":["https://orcid.org/0000-0002-9348-7262"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Zhongwen","raw_affiliation_strings":["Equal contribution"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Equal contribution","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5120050250","display_name":"Zihan Ding","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Zihan","raw_affiliation_strings":["Equal contribution"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Equal contribution","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":33,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.798799991607666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.798799991607666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.054999999701976776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.02019999921321869,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/policy-gradient-optimization","display_name":"policy gradient optimization","score":0.8360000252723694},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9380000233650208},{"id":"https://openalex.org/keywords/baseline-estimation","display_name":"baseline estimation","score":0.6679999828338623},{"id":"https://openalex.org/keywords/group-relative-policy-optimization","display_name":"Group Relative Policy Optimization","score":0.8059999942779541},{"id":"https://openalex.org/keywords/adaptive-curriculum","display_name":"adaptive curriculum","score":0.593999981880188},{"id":"https://openalex.org/keywords/llm-reasoning","display_name":"LLM reasoning","score":0.734000027179718},{"id":"https://openalex.org/keywords/mathematical-problem-solving","display_name":"mathematical problem solving","score":0.7379999756813049},{"id":"https://openalex.org/keywords/qwen3-8b","display_name":"Qwen3-8B","score":0.6610000133514404}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5938000082969666},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5401999950408936},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.4880000054836273},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4844000041484833},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.4652999937534332},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.45159998536109924},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3815000057220459},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.3725999891757965},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.36579999327659607},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.3587999939918518},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3538999855518341},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.34040001034736633},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.3255000114440918},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.31709998846054077},{"id":"https://openalex.org/C72319582","wikidata":"https://www.wikidata.org/wiki/Q584304","display_name":"Degenerate energy levels","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C62644790","wikidata":"https://www.wikidata.org/wiki/Q3454689","display_name":"Variance reduction","level":3,"score":0.2971999943256378},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2953999936580658},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2915000021457672},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28299999237060547},{"id":"https://openalex.org/C17923572","wikidata":"https://www.wikidata.org/wiki/Q7250160","display_name":"Propensity score matching","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.2766000032424927},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2581000030040741}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2509.13232","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.13232","pdf_url":"https://arxiv.org/pdf/2509.13232","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2509.13232","doi":"https://doi.org/10.48550/arxiv.2509.13232","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.13232","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2509.13232","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.13232","pdf_url":"https://arxiv.org/pdf/2509.13232","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415317764.pdf","grobid_xml":"https://content.openalex.org/works/W4415317764.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W2201581102","https://openalex.org/W2736601468","https://openalex.org/W3035700320","https://openalex.org/W3177813494","https://openalex.org/W4391631327","https://openalex.org/W4402672014","https://openalex.org/W4403883066","https://openalex.org/W4404987838","https://openalex.org/W4405904217","https://openalex.org/W4406774573","https://openalex.org/W4414449423","https://openalex.org/W4414887415","https://openalex.org/W4414939355","https://openalex.org/W4414989772","https://openalex.org/W4415036193","https://openalex.org/W4415108556","https://openalex.org/W4415109214","https://openalex.org/W4415130808","https://openalex.org/W4415137587","https://openalex.org/W4415194584","https://openalex.org/W4416048648","https://openalex.org/W4416050792","https://openalex.org/W4416070632","https://openalex.org/W4416176141","https://openalex.org/W4416243481","https://openalex.org/W4416451863","https://openalex.org/W4416540656","https://openalex.org/W4416600223","https://openalex.org/W4416610439","https://openalex.org/W4417166438","https://openalex.org/W6884962196","https://openalex.org/W6929370958","https://openalex.org/W6967114952","https://openalex.org/W7077896047","https://openalex.org/W7196953391"],"related_works":["https://openalex.org/W4416774044","https://openalex.org/W7083692674","https://openalex.org/W4414971039","https://openalex.org/W4417308124","https://openalex.org/W7133874890","https://openalex.org/W7163048680","https://openalex.org/W7111055350","https://openalex.org/W7160659004","https://openalex.org/W7203879367","https://openalex.org/W4416937012"],"abstract_inverted_index":{"We":[0,39],"revisit":[1],"policy-gradient":[2],"optimization":[3],"for":[4,74,154],"Large":[5],"Language":[6],"Models":[7],"(LLMs)":[8],"from":[9,25,137],"a":[10,56,69,148,228],"single-stream":[11],"perspective.":[12],"Prevailing":[13],"group-based":[14],"methods":[15],"like":[16],"GRPO":[17],"reduce":[18],"variance":[19],"with":[20,55,162],"on-the-fly":[21],"baselines":[22,54],"but":[23],"suffer":[24],"critical":[26],"flaws:":[27],"frequent":[28],"degenerate":[29,128],"groups":[30],"erase":[31],"learning":[32,72],"signals,":[33],"and":[34,61,83,117,144,151,202],"synchronization":[35],"barriers":[36],"hinder":[37],"scalability.":[38],"introduce":[40],"Single-stream":[41],"Policy":[42],"Optimization":[43],"(SPO),":[44],"which":[45],"eliminates":[46],"these":[47],"issues":[48],"by":[49,170,178],"design.":[50],"SPO":[51,79,113,165],"replaces":[52],"per-group":[53],"persistent,":[57],"KL-adaptive":[58],"value":[59,98],"tracker":[60,99],"normalizes":[62],"advantages":[63],"globally":[64],"across":[65,209],"the":[66,96,167,210,217,237],"batch,":[67],"providing":[68],"stable,":[70],"low-variance":[71],"signal":[73],"every":[75],"sample.":[76],"Being":[77],"group-free,":[78],"enables":[80,101],"higher":[81,119],"throughput":[82],"scales":[84],"effectively":[85],"in":[86,207,242],"long-horizon":[87],"or":[88],"tool-integrated":[89],"settings":[90],"where":[91,230],"generation":[92],"times":[93],"vary.":[94],"Furthermore,":[95],"persistent":[97],"naturally":[100],"an":[102],"adaptive":[103],"curriculum":[104],"via":[105],"prioritized":[106],"sampling.":[107],"Experiments":[108],"using":[109],"Qwen3-8B":[110],"show":[111],"that":[112,133],"converges":[114],"more":[115,149],"smoothly":[116],"attains":[118],"accuracy":[120],"than":[121],"GRPO,":[122,176],"while":[123],"eliminating":[124],"computation":[125],"wasted":[126],"on":[127,183,189,194,199],"groups.":[129],"Ablation":[130],"studies":[131],"confirm":[132],"SPO's":[134,214],"gains":[135,182],"stem":[136],"its":[138],"principled":[139],"approach":[140],"to":[141,224],"baseline":[142],"estimation":[143],"advantage":[145],"normalization,":[146],"offering":[147],"robust":[150],"efficient":[152],"path":[153,229],"LLM":[155,243],"reasoning.":[156,244],"Across":[157],"five":[158],"hard":[159],"math":[160],"benchmarks":[161],"Qwen3":[163],"8B,":[164],"improves":[166],"average":[168],"maj@32":[169],"+3.4":[171],"percentage":[172],"points":[173],"(pp)":[174],"over":[175],"driven":[177],"substantial":[179],"absolute":[180],"point":[181],"challenging":[184],"datasets,":[185],"including":[186],"+7.3":[187],"pp":[188,193,198],"BRUMO":[190],"25,":[191,196,201],"+4.4":[192],"AIME":[195],"+3.3":[197],"HMMT":[200],"achieves":[203],"consistent":[204],"relative":[205],"gain":[206],"pass@$k$":[208],"evaluated":[211],"$k$":[212],"values.":[213],"success":[215],"challenges":[216],"prevailing":[218],"trend":[219],"of":[220,240],"adding":[221],"incidental":[222],"complexity":[223],"RL":[225],"algorithms,":[226],"highlighting":[227],"fundamental":[231],"principles,":[232],"not":[233],"architectural":[234],"workarounds,":[235],"drive":[236],"next":[238],"wave":[239],"progress":[241]},"counts_by_year":[{"year":2026,"cited_by_count":28},{"year":2025,"cited_by_count":5}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-10-18T00:00:00"}