{"id":"https://openalex.org/W7225177604","doi":"https://doi.org/10.48550/arxiv.2611.03146","title":"Controlling Sharpening in LLM Post-Training","display_name":"Controlling Sharpening in LLM Post-Training","publication_year":2026,"publication_date":"2026-11-02","ids":{"openalex":"https://openalex.org/W7225177604","doi":"https://doi.org/10.48550/arxiv.2611.03146"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2611.03146","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03146","pdf_url":"https://arxiv.org/pdf/2611.03146","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2611.03146","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120549807","display_name":"Simon Matrenok","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Matrenok, Simon","raw_affiliation_strings":["School of Computer and Communication Sciences (IC) , EPFL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer and Communication Sciences (IC) , EPFL","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130473863","display_name":"Nicolas Menet","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Menet, Nicolas","raw_affiliation_strings":["Department of Computer Science , ETH Z\u00fcrich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , ETH Z\u00fcrich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061093552","display_name":"Nicolas Flammarion","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Flammarion, Nicolas","raw_affiliation_strings":["School of Computer and Communication Sciences (IC) , EPFL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer and Communication Sciences (IC) , EPFL","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5160543361","display_name":"Andreas Krause","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Krause, Andreas","raw_affiliation_strings":["Department of Computer Science , ETH Z\u00fcrich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , ETH Z\u00fcrich","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.30979999899864197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.30979999899864197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.2946999967098236,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.09309999644756317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-post-training","display_name":"LLM post-training","score":0.8169999718666077},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"reinforcement learning","score":0.5899999737739563},{"id":"https://openalex.org/keywords/sharpening","display_name":"sharpening","score":0.8479999899864197},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.843999981880188},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.8889999985694885}],"concepts":[{"id":"https://openalex.org/C2781137444","wikidata":"https://www.wikidata.org/wiki/Q237105","display_name":"Sharpening","level":2,"score":0.8147000074386597},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6510999798774719},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5722000002861023},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.42399999499320984},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.39989998936653137},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3785000145435333},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3695000112056732},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35749998688697815},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3361999988555908},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.33399999141693115},{"id":"https://openalex.org/C91575142","wikidata":"https://www.wikidata.org/wiki/Q1971426","display_name":"Optimal control","level":2,"score":0.32350000739097595},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3154999911785126},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29420000314712524},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2800999879837036},{"id":"https://openalex.org/C157972887","wikidata":"https://www.wikidata.org/wiki/Q463359","display_name":"Convex optimization","level":3,"score":0.2703000009059906},{"id":"https://openalex.org/C2987595161","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Optimization algorithm","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C55660270","wikidata":"https://www.wikidata.org/wiki/Q5164377","display_name":"Constrained optimization","level":2,"score":0.2565999925136566},{"id":"https://openalex.org/C2781395549","wikidata":"https://www.wikidata.org/wiki/Q4680762","display_name":"Adaptive sampling","level":3,"score":0.25459998846054077}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2611.03146","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03146","pdf_url":"https://arxiv.org/pdf/2611.03146","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2611.03146","doi":"https://doi.org/10.48550/arxiv.2611.03146","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.03146","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2611.03146","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03146","pdf_url":"https://arxiv.org/pdf/2611.03146","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7225177604.pdf","grobid_xml":"https://content.openalex.org/works/W7225177604.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7216324602","https://openalex.org/W7164017386","https://openalex.org/W7223236452","https://openalex.org/W4415946349","https://openalex.org/W7202214502","https://openalex.org/W4414809468","https://openalex.org/W7128018134","https://openalex.org/W7126253111","https://openalex.org/W7171924886","https://openalex.org/W7172385158"],"abstract_inverted_index":{"LLM":[0],"post-training":[1],"via":[2,176],"reinforcement":[3],"learning":[4,99],"has":[5],"shown":[6],"consistent":[7],"improvements":[8],"in":[9,87],"accuracy":[10],"at":[11,228],"the":[12,45,78,222,232],"cost":[13],"of":[14,224,231],"reducing":[15],"behavioral":[16],"coverage":[17],"-":[18],"a":[19,164,229],"phenomenon":[20],"known":[21,40],"as":[22,35],"sharpening":[23,140],"that":[24,94,113,200,210],"can":[25,213],"stall":[26],"progress":[27],"on":[28,102,141,147,151,193],"difficult":[29],"tasks.":[30],"Repeated":[31],"sampling":[32],"objectives":[33,86,116,138],"such":[34],"$\\mathrm{pass@k}$":[36,83],"and":[37,61,84,196],"$\\mathrm{max@k}$":[38,85],"are":[39],"to":[41,74,174],"mitigate":[42],"sharpening,":[43],"but":[44],"mechanism":[46],"behind":[47],"it":[48,106],"remains":[49],"poorly":[50],"understood.":[51],"Moreover,":[52],"existing":[53,171],"approaches":[54],"for":[55,66,81,135,159],"$\\mathrm{@k}$":[56,95,137,156,175,211,226],"optimization":[57,70,96],"use":[58],"dedicated":[59],"algorithms,":[60],"no":[62],"general":[63],"framework":[64],"exists":[65],"adapting":[67],"established":[68],"policy":[69],"methods":[71,173,192],"from":[72],"$\\mathrm{@}1$":[73,115,172],"$\\mathrm{@k}$.":[75],"We":[76],"derive":[77],"optimal":[79,121],"policies":[80,122],"KL-regularized":[82],"closed":[88],"form.":[89],"Our":[90],"explicit":[91],"solutions":[92],"reveal":[93],"naturally":[97],"redistributes":[98],"pressure":[100,212],"based":[101],"prompt":[103],"difficulty,":[104],"directing":[105],"toward":[107],"harder":[108,148],"problems.":[109],"Crucially,":[110,207],"we":[111,154,208],"show":[112],"standard":[114],"achieve":[117],"these":[118,152],"exact":[119],"same":[120],"through":[123],"prompt-adaptive":[124,177],"regularization":[125,178],"or":[126,179],"reward":[127,180],"transformation.":[128],"Ultimately,":[129],"this":[130],"yields":[131],"an":[132],"optimizer-independent":[133],"explanation":[134],"why":[136],"reduce":[139],"easy":[142],"tasks":[143],"while":[144],"prioritizing":[145],"success":[146],"ones.":[149],"Building":[150],"insights,":[153],"introduce":[155],"Adaptive":[157],"Pressure":[158],"Policy":[160],"Optimization":[161],"(KAPPO).":[162],"Following":[163],"one-time":[165],"reference":[166],"calibration,":[167],"KAPPO":[168,201],"seamlessly":[169],"adapts":[170],"transformation,":[181],"without":[182],"otherwise":[183],"altering":[184],"their":[185],"training":[186],"procedures.":[187],"Experiments":[188],"with":[189,217],"several":[190],"RL":[191],"mathematical":[194],"reasoning":[195],"code":[197],"generation":[198],"demonstrate":[199,209],"provides":[202],"direct":[203],"control":[204],"over":[205],"sharpening.":[206],"be":[214],"fully":[215],"realized":[216],"single":[218],"completion":[219],"training,":[220],"matching":[221],"performance":[223],"multi-completion":[225],"optimizers":[227],"fraction":[230],"cost.":[233]},"counts_by_year":[],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2026-11-04T00:00:00"}