{"id":"https://openalex.org/W7166697956","doi":"https://doi.org/10.48550/arxiv.2606.29296","title":"Process Advantage Signal Shaping: A Paradigm-Agnostic Middleware for Process-Supervised RL in LLM Reasoners","display_name":"Process Advantage Signal Shaping: A Paradigm-Agnostic Middleware for Process-Supervised RL in LLM Reasoners","publication_year":2026,"publication_date":"2026-06-28","ids":{"openalex":"https://openalex.org/W7166697956","doi":"https://doi.org/10.48550/arxiv.2606.29296"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.29296","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29296","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.29296","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139636350","display_name":"Chao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139642132","display_name":"Hongtao Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Hongtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139661601","display_name":"Tao Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100902783","display_name":"Y Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Yunsheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139671536","display_name":"Ting Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Ting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139683855","display_name":"Wenbo Ding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Wenbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6629999876022339,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6629999876022339,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.07079999893903732,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.02969999983906746,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/signal","display_name":"SIGNAL (programming language)","score":0.7063999772071838},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6417999863624573},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.5734999775886536},{"id":"https://openalex.org/keywords/block","display_name":"Block (permutation group theory)","score":0.4120999872684479},{"id":"https://openalex.org/keywords/sign","display_name":"Sign (mathematics)","score":0.40310001373291016},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.3984000086784363},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.3555999994277954},{"id":"https://openalex.org/keywords/matched-filter","display_name":"Matched filter","score":0.3431999981403351}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7121000289916992},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.7063999772071838},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6417999863624573},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.5734999775886536},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.4120999872684479},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.40310001373291016},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.3984000086784363},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.38040000200271606},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.37619999051094055},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.3555999994277954},{"id":"https://openalex.org/C50151734","wikidata":"https://www.wikidata.org/wiki/Q1759577","display_name":"Matched filter","level":3,"score":0.3431999981403351},{"id":"https://openalex.org/C146749787","wikidata":"https://www.wikidata.org/wiki/Q15963867","display_name":"Discrete-time signal","level":5,"score":0.33379998803138733},{"id":"https://openalex.org/C104267543","wikidata":"https://www.wikidata.org/wiki/Q208163","display_name":"Signal processing","level":3,"score":0.328900009393692},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.3181000053882599},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30970001220703125},{"id":"https://openalex.org/C13944312","wikidata":"https://www.wikidata.org/wiki/Q7512748","display_name":"Signal-to-noise ratio (imaging)","level":2,"score":0.295199990272522},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.2768000066280365},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25450000166893005},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.2531999945640564},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.29296","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29296","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.29296","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29296","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Group":[0],"Relative":[1],"Policy":[2],"Optimization":[3],"(GRPO)":[4],"is":[5,32],"a":[6,33,45,91,121,188,200,218],"default":[7],"recipe":[8],"for":[9],"process-supervised":[10],"reinforcement":[11],"learning":[12],"of":[13,50,77,84,111],"LLM":[14],"reasoners,":[15],"and":[16,66,81,90,132,136,161,167,183,194,208],"dense":[17],"process":[18,23,79,130],"supervision":[19],"--":[20,31,187,207],"via":[21],"learned":[22,189],"reward":[24],"models":[25],"(PRMs)":[26],"or":[27,103],"on-policy-distillation":[28,196],"KL":[29,197],"signals":[30],"common":[34],"way":[35],"to":[36],"densify":[37],"its":[38],"otherwise":[39],"weak":[40],"outcome":[41],"reward.":[42],"Layering":[43],"such":[44],"step-level":[46,129],"signal":[47,80,131,159,198],"on":[48,107,191,203],"top":[49],"GRPO's":[51,96,133],"group-standardized":[52],"advantage,":[53],"however,":[54],"exposes":[55],"three":[56,139,147],"structural":[57],"pathologies:":[58],"\\emph{channel":[59],"contamination}":[60],"between":[61,74,126],"the":[62,75,78,82,85,108,112,138,146,158,170,223],"pooled":[63],"process,":[64],"outcome,":[65],"format":[67],"streams":[68,148],"at":[69],"group":[70],"standardization;":[71],"\\emph{resolution":[72],"mismatch}":[73],"granularity":[76,83],"logical":[86],"decisions":[87],"being":[88],"credited;":[89],"\\emph{cumulative":[92],"trap}":[93],"by":[94],"which":[95],"return-to-go":[97],"sum":[98],"surfaces":[99],"either":[100],"length":[101],"inflation":[102],"truncated":[104],"exploration":[105],"depending":[106],"sign":[109],"regime":[110,215],"signal.":[113],"We":[114,177],"propose":[115],"\\textbf{PASS}":[116],"(\\emph{Process":[117],"Advantage":[118],"Signal":[119],"Shaping}),":[120],"compact":[122],"middleware":[123],"that":[124],"sits":[125],"any":[127],"scalar":[128],"clipped":[134],"surrogate":[135],"addresses":[137],"pathologies":[140],"in":[141],"turn:":[142],"\\emph{Advantage":[143],"Fusion}":[144],"standardizes":[145],"independently":[149],"within":[150,164],"each":[151,165],"group,":[152],"\\emph{Chunk-by-Value}":[153],"derives":[154],"value-homogeneous":[155],"chunks":[156],"from":[157],"itself":[160],"broadcasts":[162],"credit":[163],"chunk,":[166],"\\emph{Divide-Length}":[168],"converts":[169],"cumulative":[171],"objective":[172],"into":[173],"an":[174,195],"average-value-density":[175],"score.":[176],"validate":[178],"PASS":[179,216],"across":[180],"two":[181,184,210],"domains":[182],"process-signal":[185],"paradigms":[186],"PRM":[190],"mathematical":[192],"reasoning":[193],"(with":[199],"generalized":[201],"variant)":[202],"multi-hop":[204],"question":[205],"answering":[206],"under":[209],"group-standardization":[211],"operators.":[212],"In":[213],"every":[214],"delivers":[217],"consistent":[219],"pass@1":[220],"gain":[221],"over":[222],"corresponding":[224],"GRPO":[225],"baseline.":[226]},"counts_by_year":[],"updated_date":"2026-07-01T06:29:00.853634","created_date":"2026-07-01T00:00:00"}