{"id":"https://openalex.org/W7129243956","doi":"https://doi.org/10.48550/arxiv.2602.13659","title":"Zero-Order Optimization for LLM Fine-Tuning via Learnable Direction Sampling","display_name":"Zero-Order Optimization for LLM Fine-Tuning via Learnable Direction Sampling","publication_year":2026,"publication_date":"2026-02-14","ids":{"openalex":"https://openalex.org/W7129243956","doi":"https://doi.org/10.48550/arxiv.2602.13659"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.13659","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031204733","display_name":"\u0412. \u0415. \u041f\u0430\u0440\u0444\u0435\u043d\u043e\u0432","orcid":"https://orcid.org/0000-0002-3221-5466"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Parfenov, Valery","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002296396","display_name":"Grigoriy Evseev","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Evseev, Grigoriy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093118722","display_name":"Andrey Veprikov","orcid":"https://orcid.org/0009-0001-9126-572X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Veprikov, Andrey","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126257273","display_name":"Nikolay Bushkov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bushkov, Nikolay","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126207472","display_name":"Stanislav Moiseev","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moiseev, Stanislav","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5126227695","display_name":"Aleksandr Beznosikov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Beznosikov, Aleksandr","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5031204733"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.21699999272823334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.21699999272823334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.14900000393390656,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.13699999451637268,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.7283999919891357},{"id":"https://openalex.org/keywords/curse-of-dimensionality","display_name":"Curse of dimensionality","score":0.6593000292778015},{"id":"https://openalex.org/keywords/backpropagation","display_name":"Backpropagation","score":0.5867000222206116},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.5645999908447266},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5636000037193298},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5346999764442444},{"id":"https://openalex.org/keywords/importance-sampling","display_name":"Importance sampling","score":0.4772999882698059},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.4262000024318695},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.38519999384880066}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.7283999919891357},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.6593000292778015},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6082000136375427},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.5867000222206116},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.5645999908447266},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5636000037193298},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5346999764442444},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5006999969482422},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.4772999882698059},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4683000147342682},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.4262000024318695},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.38519999384880066},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.36640000343322754},{"id":"https://openalex.org/C2781395549","wikidata":"https://www.wikidata.org/wiki/Q4680762","display_name":"Adaptive sampling","level":3,"score":0.35510000586509705},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.34139999747276306},{"id":"https://openalex.org/C1893757","wikidata":"https://www.wikidata.org/wiki/Q3653001","display_name":"Inversion (geology)","level":3,"score":0.3343999981880188},{"id":"https://openalex.org/C167723999","wikidata":"https://www.wikidata.org/wiki/Q3773214","display_name":"Sampling distribution","level":2,"score":0.3292999863624573},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3199999928474426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2913999855518341},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.28839999437332153},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.2858000099658966},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C204241405","wikidata":"https://www.wikidata.org/wiki/Q461499","display_name":"Transformation (genetics)","level":3,"score":0.2703999876976013},{"id":"https://openalex.org/C115680565","wikidata":"https://www.wikidata.org/wiki/Q5977448","display_name":"Gradient method","level":2,"score":0.26660001277923584},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.2574999928474426},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2554999887943268},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2538999915122986}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.13659","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.13659","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.13659","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.13659","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Fine-tuning":[0],"large":[1,21],"pretrained":[2],"language":[3],"models":[4],"(LLMs)":[5],"is":[6,161,175],"a":[7,75,88,103,111,162],"cornerstone":[8],"of":[9,98,122],"modern":[10],"NLP,":[11],"yet":[12],"its":[13],"growing":[14],"memory":[15,43],"demands":[16],"(driven":[17],"by":[18,34],"backpropagation":[19,33],"and":[20,53,91,109,125],"optimizer":[22],"States)":[23],"limit":[24],"deployment":[25],"in":[26,132],"resource-constrained":[27],"settings.":[28],"Zero-order":[29],"(ZO)":[30],"methods":[31],"bypass":[32],"estimating":[35],"directional":[36,99],"derivatives":[37],"from":[38,50],"forward":[39],"evaluations,":[40],"offering":[41],"substantial":[42],"savings.":[44],"However,":[45],"classical":[46],"ZO":[47,77,152,167],"estimators":[48],"suffer":[49],"high":[51],"variance":[52,97],"an":[54],"adverse":[55],"dependence":[56,129],"on":[57,130,140],"the":[58,81,96,120,127,138],"parameter":[59],"dimensionality":[60],"$d$,":[61],"which":[62],"has":[63],"constrained":[64],"their":[65],"use":[66],"to":[67,94,150,165],"low-dimensional":[68],"problems.":[69],"In":[70],"this":[71,107],"work,":[72],"we":[73,136],"propose":[74],"policy-driven":[76],"framework":[78],"that":[79,115,157],"treats":[80],"sampling":[82,117,160],"distribution":[83],"over":[84],"perturbation":[85],"directions":[86],"as":[87],"learnable":[89],"policy":[90],"updates":[92],"it":[93],"reduce":[95],"estimates.":[100],"We":[101],"develop":[102],"practical":[104],"algorithm":[105],"implementing":[106],"idea":[108],"provide":[110],"theoretical":[112],"analysis,":[113],"showing":[114],"learned":[116],"distributions":[118],"improve":[119],"quality":[121],"gradient":[123],"information":[124],"relax":[126],"explicit":[128],"$d$":[131],"convergence":[133],"bounds.":[134],"Empirically,":[135],"validate":[137],"approach":[139],"challenging":[141],"LLM":[142],"fine-tuning":[143,168],"benchmarks,":[144],"demonstrating":[145],"substantially":[146],"improved":[147],"performance":[148],"compared":[149],"standard":[151],"baselines.":[153],"Our":[154],"results":[155],"suggest":[156],"adaptive":[158],"direction":[159],"promising":[163],"route":[164],"make":[166],"viable":[169],"at":[170,177],"scale.":[171],"The":[172],"source":[173],"code":[174],"available":[176],"https://github.com/brain-lab-research/zo_ldsd":[178]},"counts_by_year":[],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2026-02-18T00:00:00"}