{"id":"https://openalex.org/W7166710770","doi":"https://doi.org/10.48550/arxiv.2606.29526","title":"The Mirage of Optimizing Training Policies: Monotonic Inference Policies as the Real Objective for LLM Reinforcement Learning","display_name":"The Mirage of Optimizing Training Policies: Monotonic Inference Policies as the Real Objective for LLM Reinforcement Learning","publication_year":2026,"publication_date":"2026-06-28","ids":{"openalex":"https://openalex.org/W7166710770","doi":"https://doi.org/10.48550/arxiv.2606.29526"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.29526","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29526","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.29526","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139640585","display_name":"Jing Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Jing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139681676","display_name":"Hongyao Tang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Hongyao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139660714","display_name":"Yi Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121999607","display_name":"Y He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yancheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139693187","display_name":"Weixun Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Weixun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139644284","display_name":"Xiaoyang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiaoyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006022044","display_name":"J W Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Ju","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139703361","display_name":"Wenbo Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Wenbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139663444","display_name":"Jinyi Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jinyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139636751","display_name":"Yan Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139637323","display_name":"Jianye Hao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hao, Jianye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139643588","display_name":"Bo Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.249099999666214,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.249099999666214,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.14640000462532043,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.08399999886751175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.8115000128746033},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6559000015258789},{"id":"https://openalex.org/keywords/monotonic-function","display_name":"Monotonic function","score":0.6274999976158142},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.616100013256073},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5389000177383423},{"id":"https://openalex.org/keywords/inference-engine","display_name":"Inference engine","score":0.4275999963283539}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.8115000128746033},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.692799985408783},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6559000015258789},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.6274999976158142},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.616100013256073},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5861999988555908},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5476999878883362},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5389000177383423},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.4275999963283539},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3230000138282776},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.27970001101493835},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.2784999907016754}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.29526","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29526","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.29526","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29526","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"(RL)":[2],"has":[3],"gained":[4],"growing":[5],"attention":[6],"in":[7,45,84,116,133],"large":[8],"language":[9],"model":[10,62,188],"(LLM)":[11],"post-training,":[12],"yet":[13],"RL":[14,167],"training":[15,36,42,55,91,118,201],"remains":[16],"fragile":[17],"and":[18,35,41,56,74,174,200],"can":[19],"suffer":[20],"from":[21],"instability":[22],"or":[23],"collapse.":[24],"One":[25],"vital":[26],"cause":[27],"is":[28],"training-inference":[29],"mismatch:":[30],"LLM":[31,146,166],"adopts":[32],"separate":[33],"inference":[34,57,127],"engines":[37],"for":[38,50,145],"generation":[39],"efficiency":[40],"precision,":[43],"which":[44],"practice":[46],"exhibits":[47],"inconsistent":[48],"probabilities":[49],"the":[51,76,86,90,94,102,114,117,123,126,130],"same":[52],"trajectories":[53],"on":[54,186],"sides,":[58],"even":[59],"with":[60],"synchronized":[61,177],"parameters.":[63],"This":[64],"naturally":[65],"induces":[66],"a":[67,140,164],"special":[68],"type":[69],"of":[70,125],"off-policyness":[71,87],"ever":[72],"existing":[73,107],"poisoning":[75],"training.":[77],"Prior":[78],"works":[79,108],"have":[80],"made":[81],"various":[82],"efforts":[83],"addressing":[85],"to":[88,113],"stabilize":[89],"policies":[92],"under":[93,190],"mismatch.":[95],"In":[96],"this":[97,136,155],"paper,":[98],"we":[99,138,157],"point":[100],"out":[101],"objective":[103,144],"misalignment":[104],"neglected":[105],"by":[106],"that":[109,169,194],"an":[110,180],"effective":[111],"update":[112],"policy":[115,142],"engine":[119],"not":[120],"necessarily":[121],"ensures":[122],"improvement":[124],"policy,":[128],"i.e.,":[129],"one":[131],"used":[132],"deployment.":[134],"To":[135],"end,":[137],"propose":[139],"new":[141],"optimization":[143],"RL,":[147],"named":[148],"Monotonic":[149,159],"Inference":[150,160],"Policy":[151,161],"Improvement":[152],"(MIPI).":[153],"Following":[154],"principle,":[156],"introduce":[158],"Update":[162],"(MIPU),":[163],"two-step":[165],"framework":[168],"constructs":[170],"sampler-referenced":[171],"candidate":[172],"updates":[173],"selectively":[175],"accepts":[176],"candidates":[178],"using":[179],"inference-side":[181],"gap":[182],"proxy.":[183],"Experiments":[184],"conducted":[185],"two":[187],"scales":[189],"high":[191],"mismatch":[192],"show":[193],"MIPU":[195],"improves":[196],"average":[197],"reasoning":[198],"performance":[199],"stability.":[202]},"counts_by_year":[],"updated_date":"2026-07-01T06:29:00.853634","created_date":"2026-07-01T00:00:00"}