{"id":"https://openalex.org/W7128740461","doi":"https://doi.org/10.48448/qnvf-5346","title":"PSPO: Prompt-Level Prioritization and Experience-Weighted Smoothing for Efficient Policy Optimization","display_name":"PSPO: Prompt-Level Prioritization and Experience-Weighted Smoothing for Efficient Policy Optimization","publication_year":2026,"publication_date":"2026-01-07","ids":{"openalex":"https://openalex.org/W7128740461","doi":"https://doi.org/10.48448/qnvf-5346"},"language":"en","primary_location":{"id":"doi:10.48448/qnvf-5346","doi":"https://doi.org/10.48448/qnvf-5346","is_oa":true,"landing_page_url":"https://doi.org/10.48448/qnvf-5346","pdf_url":null,"source":{"id":"https://openalex.org/S7407053148","display_name":"Underline Science Inc.","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"volume":null,"type":"other","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48448/qnvf-5346","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125741655","display_name":"Association for Artificial Intelligence 2026","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Association for Artificial Intelligence 2026","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102620260","display_name":"Jiongfeng Fang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Jiongfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108312654","display_name":"Haowen Hou","orcid":"https://orcid.org/0009-0002-6944-0939","observed_orcids":["https://orcid.org/0009-0002-6944-0939"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hou, Haowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125757328","display_name":"Yulin Peng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Yulin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125744457","display_name":"Ying Tiffany He","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tiffany He, Ying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125694214","display_name":"Fei Yu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Fei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125726023","display_name":"Nianbo Zeng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Nianbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035554497","display_name":"Ruichong Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ruichong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130102815","display_name":"Xinxin Zhu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Xinxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9199000000953674,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9199000000953674,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.018699999898672104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.009100000374019146,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-fine-tuning","display_name":"reinforcement fine-tuning","score":0.7870000004768372},{"id":"https://openalex.org/keywords/proximal-policy-optimization","display_name":"proximal policy optimization","score":0.8240000009536743},{"id":"https://openalex.org/keywords/group-relative-policy-optimization","display_name":"Group Relative Policy Optimization","score":0.9769999980926514},{"id":"https://openalex.org/keywords/exponentially-weighted-moving-average","display_name":"exponential moving average","score":0.6990000009536743},{"id":"https://openalex.org/keywords/advantage-estimation","display_name":"advantage estimation","score":0.6570000052452087},{"id":"https://openalex.org/keywords/prioritized-experience-replay","display_name":"prioritized experience replay","score":0.7490000128746033},{"id":"https://openalex.org/keywords/sample-efficiency","display_name":"sample efficiency","score":0.7960000038146973},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.753000020980835},{"id":"https://openalex.org/keywords/preference-alignment","display_name":"preference alignment","score":0.6919999718666077}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8043000102043152},{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.6793000102043152},{"id":"https://openalex.org/C133710760","wikidata":"https://www.wikidata.org/wiki/Q775837","display_name":"Exponential smoothing","level":2,"score":0.6255999803543091},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5185999870300293},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5060999989509583},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4830000102519989},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46549999713897705},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.41119998693466187},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40220001339912415},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.40130001306533813},{"id":"https://openalex.org/C2777615720","wikidata":"https://www.wikidata.org/wiki/Q11888847","display_name":"Prioritization","level":2,"score":0.39419999718666077},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.36550000309944153},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.35420000553131104},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.3190000057220459},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3122999966144562},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.30799999833106995},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.29989999532699585},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.274399995803833},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.272599995136261},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.25459998846054077}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48448/qnvf-5346","doi":"https://doi.org/10.48448/qnvf-5346","is_oa":true,"landing_page_url":"https://doi.org/10.48448/qnvf-5346","pdf_url":null,"source":{"id":"https://openalex.org/S7407053148","display_name":"Underline Science Inc.","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"}],"best_oa_location":{"id":"doi:10.48448/qnvf-5346","doi":"https://doi.org/10.48448/qnvf-5346","is_oa":true,"landing_page_url":"https://doi.org/10.48448/qnvf-5346","pdf_url":null,"source":{"id":"https://openalex.org/S7407053148","display_name":"Underline Science Inc.","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4416937012","https://openalex.org/W4417308124","https://openalex.org/W4414971039","https://openalex.org/W4415036193","https://openalex.org/W4414810188","https://openalex.org/W4414791254","https://openalex.org/W7133874890","https://openalex.org/W7169596520","https://openalex.org/W4416052723","https://openalex.org/W7119492051"],"abstract_inverted_index":{"Reinforcement":[0],"Fine-tuning":[1],"(RFT)":[2],"methods":[3],"such":[4],"as":[5],"Proximal":[6],"Policy":[7,13,56],"Optimization":[8,14],"(PPO)":[9],"and":[10,51,69,121,164,180,184],"Group":[11],"Relative":[12],"(GRPO)":[15],"have":[16],"demonstrated":[17],"strong":[18],"capabilities":[19],"in":[20,118],"aligning":[21],"Large":[22],"Language":[23],"Models":[24],"(LLMs)":[25],"with":[26],"human":[27],"preferences.":[28],"However,":[29],"these":[30],"approaches":[31],"often":[32],"suffer":[33],"from":[34],"limited":[35],"data":[36,136,156],"efficiency,":[37],"necessitating":[38],"extensive":[39],"on-policy":[40],"rollouts":[41],"to":[42,63,89,113],"maintain":[43],"competitive":[44],"performance.":[45],"We":[46],"propose":[47],"PSPO":[48,168],"(Prompt-Level":[49],"Prioritization":[50],"Experience-Weighted":[52],"Smoothing":[53],"for":[54,94],"Efficient":[55],"Optimization),":[57],"a":[58,119,127],"lightweight":[59,120],"yet":[60],"effective":[61],"enhancement":[62],"GRPO":[64],"that":[65,167],"improves":[66],"training":[67,104],"stability":[68],"sample":[70],"efficiency":[71],"through":[72],"two":[73],"complementary":[74],"techniques.":[75],"First,":[76],"we":[77,125],"introduce":[78],"an":[79,134],"experience-weighted":[80],"reward":[81,92,116],"smoothing":[82],"mechanism,":[83],"which":[84,132],"uses":[85],"exponential":[86],"moving":[87],"averages":[88],"track":[90],"group-level":[91],"statistics":[93],"each":[95],"prompt.":[96],"This":[97],"enables":[98],"more":[99],"stable":[100],"advantage":[101],"estimation":[102],"across":[103],"steps":[105],"without":[106],"storing":[107],"entire":[108],"trajectories,":[109],"allowing":[110],"the":[111],"model":[112],"capture":[114],"historical":[115],"trends":[117],"memory-efficient":[122],"manner.":[123],"Second,":[124],"adopt":[126],"prompt-level":[128],"prioritized":[129,141],"sampling":[130],"strategy,":[131],"is":[133],"online":[135],"selection":[137],"method":[138],"inspired":[139],"by":[140],"experience":[142],"replay.":[143],"It":[144],"dynamically":[145],"emphasizes":[146],"higher-impact":[147],"prompts":[148],"based":[149],"on":[150,159],"their":[151],"relative":[152],"advantages,":[153],"thereby":[154],"improving":[155],"efficiency.":[157],"Experiments":[158],"multiple":[160],"mathematical":[161],"reasoning":[162],"benchmarks":[163],"models":[165],"show":[166],"achieves":[169],"comparable":[170],"or":[171],"better":[172],"accuracy":[173],"than":[174],"GRPO,":[175],"while":[176],"significantly":[177],"accelerating":[178],"convergence,":[179],"maintaining":[181],"low":[182],"computational":[183],"memory":[185],"overhead.":[186]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-02-13T00:00:00"}