{"id":"https://openalex.org/W7166870771","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1767","title":"Save the Good Prefix: Precise Error Penalization via Process-Supervised RL to Enhance LLM Reasoning","display_name":"Save the Good Prefix: Precise Error Penalization via Process-Supervised RL to Enhance LLM Reasoning","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166870771","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1767"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.1767","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1767","pdf_url":"https://aclanthology.org/2026.findings-acl.1767.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-acl.1767.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139784294","display_name":"Haolin Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haolin Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139836704","display_name":"Dian Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dian Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028846454","display_name":"Sidi Lu","orcid":"https://orcid.org/0000-0001-9846-7570"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sidi Lu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139765177","display_name":"Yujun Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yujun Zhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139771431","display_name":"Rui Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rui Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139827657","display_name":"Zhenwen Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhenwen Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139807314","display_name":"Haitao Mi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haitao Mi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124055101","display_name":"Chen-Yu Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen-Yu Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139725856","display_name":"Dong Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.85932014,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"35450","last_page":"35477"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5753999948501587,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5753999948501587,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13930000364780426,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11010","display_name":"Logic, Reasoning, and Knowledge","score":0.0478999987244606,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.2556000053882599},{"id":"https://openalex.org/keywords/control-theory","display_name":"Control theory (sociology)","score":0.24779999256134033},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.22669999301433563},{"id":"https://openalex.org/keywords/case-based-reasoning","display_name":"Case-based reasoning","score":0.22010000050067902}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5443999767303467},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3488999903202057},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.289000004529953},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2824999988079071},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.24779999256134033},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.24629999697208405},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.22669999301433563},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.22010000050067902},{"id":"https://openalex.org/C2776639384","wikidata":"https://www.wikidata.org/wiki/Q840396","display_name":"Ideal (ethics)","level":2,"score":0.21789999306201935}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.1767","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1767","pdf_url":"https://aclanthology.org/2026.findings-acl.1767.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.1767","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1767","pdf_url":"https://aclanthology.org/2026.findings-acl.1767.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166870771.pdf","grobid_xml":"https://content.openalex.org/works/W7166870771.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1,158],"(RL)":[2],"has":[3],"emerged":[4],"as":[5,95],"a":[6,56,63,73,129],"powerful":[7],"framework":[8],"for":[9],"improving":[10],"the":[11,68,116,126,139,143,151],"reasoning":[12,74,165],"capabilities":[13],"of":[14],"large":[15],"language":[16],"models":[17,40],"(LLMs).However,":[18],"most":[19],"existing":[20],"RL":[21,171],"approaches":[22],"rely":[23],"on":[24,62,138,176],"sparse":[25],"outcome":[26],"rewards,":[27],"which":[28,110],"fail":[29],"to":[30,54,98,114],"credit":[31,162],"correct":[32,131],"intermediate":[33],"steps":[34],"in":[35,72,87],"partially":[36],"successful":[37],"solutions.Process":[38],"reward":[39],"(PRMs)":[41],"offer":[42],"fine-grained":[43],"step-level":[44],"supervision,":[45],"but":[46],"their":[47,90],"scores":[48,92],"are":[49,84,93],"often":[50],"noisy":[51],"and":[52,133,160,172,179],"difficult":[53],"evaluate.As":[55],"result,":[57],"recent":[58],"PRM":[59],"benchmarks":[60],"focus":[61],"more":[64],"objective":[65],"capability:":[66],"detecting":[67],"first":[69,117,140],"incorrect":[70,122],"step":[71],"path.However,":[75],"this":[76,101],"evaluation":[77],"target":[78],"is":[79],"misaligned":[80],"with":[81],"how":[82],"PRMs":[83,112],"typically":[85],"used":[86],"RL,":[88],"where":[89],"step-wise":[91],"treated":[94],"raw":[96],"rewards":[97],"maximize.To":[99],"bridge":[100],"gap,":[102],"we":[103],"propose":[104],"Verifiable":[105],"Prefix":[106],"Policy":[107],"Optimization":[108],"(VPPO),":[109],"uses":[111],"only":[113,149],"localize":[115],"error":[118],"during":[119],"RL.Given":[120],"an":[121,134],"rollout,":[123],"VPPO":[124,167],"partitions":[125],"trajectory":[127],"into":[128],"verified":[130],"prefix":[132],"erroneous":[135],"suffix":[136],"based":[137],"error,":[141],"rewarding":[142],"former":[144],"while":[145],"applying":[146],"targeted":[147],"penalties":[148],"after":[150],"detected":[152],"mistake.This":[153],"design":[154],"yields":[155],"stable,":[156],"interpretable":[157],"signals":[159],"improves":[161],"assignment.Across":[163],"multiple":[164],"benchmarks,":[166],"consistently":[168],"outperforms":[169],"sparse-reward":[170],"prior":[173],"PRM-guided":[174],"baselines":[175],"both":[177],"Pass@1":[178],"Pass@K.":[180]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}