{"id":"https://openalex.org/W7225167252","doi":"https://doi.org/10.48550/arxiv.2611.03260","title":"The Cost of Mismatch: Noise Amplification in Zeroth-Order Reinforcement Learning","display_name":"The Cost of Mismatch: Noise Amplification in Zeroth-Order Reinforcement Learning","publication_year":2026,"publication_date":"2026-11-02","ids":{"openalex":"https://openalex.org/W7225167252","doi":"https://doi.org/10.48550/arxiv.2611.03260"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2611.03260","doi":"https://doi.org/10.48550/arxiv.2611.03260","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.03260","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2611.03260","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090934537","display_name":"\u9648\u8fde\u6c11","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Lianmin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027871414","display_name":"Junbin Qiu","orcid":"https://orcid.org/0000-0003-0589-822X","observed_orcids":["https://orcid.org/0000-0003-0589-822X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Junbin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160563663","display_name":"Chenxing Wei","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Chenxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000956644","display_name":"Yao Shu","orcid":"https://orcid.org/0000-0001-8847-0549","observed_orcids":["https://orcid.org/0000-0001-8847-0549"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shu, Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5160562463","display_name":"Kun He","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Kun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9746000170707703,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9746000170707703,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.00930000003427267,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/derivative-free-optimization","display_name":"derivative-free optimization","score":0.9300000071525574},{"id":"https://openalex.org/keywords/noise-amplification","display_name":"noise amplification","score":0.7699999809265137},{"id":"https://openalex.org/keywords/reinforcement-learning-with-verifiable-rewards","display_name":"reinforcement learning with verifiable rewards","score":0.8119999766349792},{"id":"https://openalex.org/keywords/matched-data","display_name":"matched data","score":0.3959999978542328},{"id":"https://openalex.org/keywords/analysis-of-variance","display_name":"analysis of variance","score":0.7390000224113464},{"id":"https://openalex.org/keywords/convergence-guarantees","display_name":"convergence guarantees","score":0.699999988079071},{"id":"https://openalex.org/keywords/memory-efficient-fine-tuning","display_name":"memory-efficient fine-tuning","score":0.6669999957084656}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7993999719619751},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6389999985694885},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5849999785423279},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5199999809265137},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4749000072479248},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.44940000772476196},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.42399999499320984},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.4171999990940094},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3928999900817871},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39079999923706055},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3781999945640564},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.37279999256134033},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36550000309944153},{"id":"https://openalex.org/C2780909371","wikidata":"https://www.wikidata.org/wiki/Q4801092","display_name":"Artificial noise","level":4,"score":0.3409999907016754},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3257000148296356},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3093999922275543},{"id":"https://openalex.org/C29265498","wikidata":"https://www.wikidata.org/wiki/Q7047719","display_name":"Noise measurement","level":3,"score":0.30000001192092896},{"id":"https://openalex.org/C35304006","wikidata":"https://www.wikidata.org/wiki/Q5962","display_name":"Boltzmann constant","level":2,"score":0.2946999967098236},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.2921999990940094},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.28619998693466187},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2809999883174896},{"id":"https://openalex.org/C2777317252","wikidata":"https://www.wikidata.org/wiki/Q18393516","display_name":"Rare events","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.2694999873638153},{"id":"https://openalex.org/C2987595161","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Optimization algorithm","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C2986577269","wikidata":"https://www.wikidata.org/wiki/Q11306265","display_name":"Random noise","level":2,"score":0.2565999925136566}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2611.03260","doi":"https://doi.org/10.48550/arxiv.2611.03260","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.03260","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2611.03260","doi":"https://doi.org/10.48550/arxiv.2611.03260","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.03260","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7083686985","https://openalex.org/W7163368891","https://openalex.org/W7118489425","https://openalex.org/W7161289579","https://openalex.org/W7224249081","https://openalex.org/W7203705598","https://openalex.org/W4415311914","https://openalex.org/W7119933302","https://openalex.org/W4414581770","https://openalex.org/W7119609575"],"abstract_inverted_index":{"Zeroth-order":[0],"(ZO)":[1],"optimization":[2,146],"has":[3],"emerged":[4],"as":[5,96],"a":[6,43,86,186],"promising":[7],"approach":[8],"for":[9,143],"memory-efficient":[10],"large":[11],"language":[12],"model":[13],"(LLM)":[14],"post-training,":[15],"yet":[16],"its":[17],"reliability":[18,52,62],"in":[19],"reinforcement":[20,78],"learning":[21,79],"with":[22,122],"verifiable":[23],"rewards":[24],"(RLVR)":[25],"remains":[26],"largely":[27],"unexplored.":[28],"Motivated":[29],"by":[30,66],"the":[31,51,68,82,97,108,128,134,159,181],"distinct":[32],"data":[33,163],"regimes":[34],"of":[35,53,147,162],"on-policy":[36],"RL":[37],"and":[38,154,169],"fixed-data":[39],"optimization,":[40],"we":[41],"develop":[42],"generic":[44],"variance-theoretic":[45],"framework":[46],"to":[47],"investigate":[48],"what":[49],"limits":[50],"ZO":[54,145],"gradient":[55],"estimation.":[56],"Our":[57],"analysis":[58],"reveals":[59],"that":[60,90,125,176],"estimation":[61],"is":[63,192],"structurally":[64],"determined":[65],"whether":[67],"two":[69,129,170],"finite-difference":[70],"evaluations":[71],"are":[72],"performed":[73],"on":[74,139],"matched":[75],"data.":[76],"On-policy":[77],"(RL)":[80],"under":[81,107],"mismatched-data":[83],"regime":[84,110],"produces":[85],"$1/\u03bc^2$":[87],"oracle-noise":[88],"amplification":[89,113],"sets":[91],"an":[92,148],"irreducible":[93],"noise":[94,115],"floor":[95],"perturbation":[98],"scale":[99],"$\u03bc$":[100],"decreases,":[101],"whereas":[102],"Boltzmann":[103],"Targeted":[104],"SFT":[105],"(BOLT)":[106],"matched-data":[109,141],"eliminates":[111],"this":[112],"via":[114],"cancellation.":[116],"We":[117],"further":[118,174],"derive":[119],"convergence":[120],"guarantees":[121],"explicit":[123],"constants":[124],"cleanly":[126],"separate":[127],"regimes.":[130],"These":[131],"results":[132],"motivate":[133],"ZO-BOLT":[135,177],"algorithm,":[136],"which":[137],"builds":[138],"BOLT's":[140],"advantage":[142],"reliable":[144],"RLVR-aligned":[149],"objective.":[150],"Controlled":[151],"perturbation-scale":[152],"sweeps":[153],"matched-versus-mismatched":[155],"end-to-end":[156],"comparisons":[157],"validate":[158],"predicted":[160],"effect":[161],"matching.":[164],"Experiments":[165],"across":[166],"three":[167],"models":[168],"mathematical":[171],"reasoning":[172],"benchmarks":[173],"show":[175],"consistently":[178],"improves":[179],"over":[180],"pretrained":[182],"baseline":[183],"while":[184],"retaining":[185],"near-inference":[187],"memory":[188],"footprint.":[189],"The":[190],"code":[191],"publicly":[193],"available":[194],"at":[195],"https://github.com/lianmin-chen/the_cost_of_mismatch.":[196]},"counts_by_year":[],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2026-11-04T00:00:00"}