{"id":"https://openalex.org/W7163030852","doi":"https://doi.org/10.48550/arxiv.2605.31494","title":"Consolidating Rewarded Perturbations for LLM Post-Training","display_name":"Consolidating Rewarded Perturbations for LLM Post-Training","publication_year":2026,"publication_date":"2026-05-29","ids":{"openalex":"https://openalex.org/W7163030852","doi":"https://doi.org/10.48550/arxiv.2605.31494"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.31494","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.31494","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.31494","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137604713","display_name":"Zheyu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zheyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137542576","display_name":"Shuo Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Shuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137537310","display_name":"Gjergji Kasneci","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kasneci, Gjergji","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5023000240325928,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5023000240325928,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.062300000339746475,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.06109999865293503,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perturbation","display_name":"Perturbation (astronomy)","score":0.5490999817848206},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.4880000054836273},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.48069998621940613},{"id":"https://openalex.org/keywords/gaussian","display_name":"Gaussian","score":0.4733000099658966},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.46160000562667847},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.43779999017715454},{"id":"https://openalex.org/keywords/operator","display_name":"Operator (biology)","score":0.4106999933719635},{"id":"https://openalex.org/keywords/control-theory","display_name":"Control theory (sociology)","score":0.3944999873638153},{"id":"https://openalex.org/keywords/cutoff","display_name":"Cutoff","score":0.38019999861717224}],"concepts":[{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.5490999817848206},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.4880000054836273},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.48069998621940613},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.4733000099658966},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.46160000562667847},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4465999901294708},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.43779999017715454},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.4106999933719635},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4034999907016754},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3993000090122223},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.3944999873638153},{"id":"https://openalex.org/C2778217198","wikidata":"https://www.wikidata.org/wiki/Q556977","display_name":"Cutoff","level":2,"score":0.38019999861717224},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3765999972820282},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.3587000072002411},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.3352999985218048},{"id":"https://openalex.org/C51865526","wikidata":"https://www.wikidata.org/wiki/Q5379962","display_name":"Ensemble average","level":2,"score":0.3246999979019165},{"id":"https://openalex.org/C186886427","wikidata":"https://www.wikidata.org/wiki/Q5441213","display_name":"Feedback loop","level":2,"score":0.32269999384880066},{"id":"https://openalex.org/C198352243","wikidata":"https://www.wikidata.org/wiki/Q37105","display_name":"Line (geometry)","level":2,"score":0.3183000087738037},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3165999948978424},{"id":"https://openalex.org/C119898033","wikidata":"https://www.wikidata.org/wiki/Q3433888","display_name":"Ensemble forecasting","level":2,"score":0.3050000071525574},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3003000020980835},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.287200003862381},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.26989999413490295},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2639999985694885},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.2624000012874603},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.25220000743865967}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.31494","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.31494","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.31494","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.31494","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7077324390411377,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Post-training":[0],"of":[1,18,170,186,189],"language":[2,138,142],"models":[3,143],"is":[4],"commonly":[5],"framed":[6],"as":[7],"a":[8,33,83,118,128],"sample-score-update":[9],"loop":[10,25],"implemented":[11],"by":[12,21,162,178],"gradient":[13,134],"descent.":[14],"A":[15,95],"recent":[16],"line":[17],"work,":[19],"exemplified":[20],"RandOpt,":[22],"relocates":[23],"this":[24,54,111],"to":[26,69,146],"weight":[27],"space,":[28],"sampling":[29],"Gaussian":[30],"perturbations":[31],"around":[32],"pretrained":[34],"model":[35,161],"and":[36,48,64,127,148,154,181],"ensembling":[37],"the":[38,75,88,137,159,187,190],"top-K":[39],"rewarded":[40,76],"specialists":[41],"at":[42,194],"inference.":[43],"While":[44],"competitive":[45],"with":[46,91,132],"PPO":[47],"GRPO":[49],"under":[50],"matched":[51],"training":[52],"compute,":[53],"prediction-level":[55],"ensemble":[56,90],"incurs":[57],"K":[58],"forward":[59,196],"passes":[60],"per":[61,198],"test":[62,199],"example":[63],"does":[65],"not":[66],"extend":[67],"cleanly":[68],"free-form":[70],"generation.":[71],"We":[72,109],"ask":[73],"whether":[74],"population":[77],"can":[78],"instead":[79],"be":[80],"folded":[81],"into":[82,113],"single":[84],"deployable":[85],"model,":[86],"replacing":[87],"inference-time":[89],"one":[92,168,195],"consolidated":[93],"update.":[94],"split-half":[96],"analysis":[97],"over":[98],"25":[99],"model-task":[100],"pairs":[101],"reveals":[102],"reproducible":[103],"low-rank":[104],"structure":[105],"in":[106],"every":[107],"case.":[108],"turn":[110],"geometry":[112],"CoRP":[114,157,174],"(Consolidating":[115],"Rewarded":[116],"Perturbations),":[117],"gradient-free":[119],"operator":[120],"that":[121],"combines":[122],"reward-weighted":[123],"aggregation,":[124],"compatibility-aware":[125],"reweighting,":[126],"held-out":[129],"validation":[130],"gate,":[131],"no":[133],"flowing":[135],"through":[136],"model.":[139],"Across":[140],"five":[141,149],"from":[144],"0.5B":[145],"8B":[147],"tasks":[150],"covering":[151],"math,":[152],"code,":[153],"creative":[155],"writing,":[156],"improves":[158],"base":[160],"8.1":[163],"points":[164,180],"on":[165],"average.":[166],"Using":[167],"tenth":[169],"RandOpt's":[171],"perturbation":[172],"budget,":[173],"exceeds":[175],"single-inference":[176],"RandOpt":[177],"6.5":[179],"recovers":[182],"more":[183],"than":[184],"half":[185],"gain":[188],"50-pass":[191],"majority-vote":[192],"ensemble,":[193],"pass":[197],"example.":[200]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-02T00:00:00"}