{"id":"https://openalex.org/W7216515338","doi":"https://doi.org/10.48550/arxiv.2610.02190","title":"Trust the Direction, Search the Step: Zero-and-First-Order Methods for LLM Fine-Tuning","display_name":"Trust the Direction, Search the Step: Zero-and-First-Order Methods for LLM Fine-Tuning","publication_year":2026,"publication_date":"2026-10-01","ids":{"openalex":"https://openalex.org/W7216515338","doi":"https://doi.org/10.48550/arxiv.2610.02190"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.02190","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02190","pdf_url":"https://arxiv.org/pdf/2610.02190","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.02190","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138642489","display_name":"Cristian McGee","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"McGee, Cristian","raw_affiliation_strings":["University of Central Florida Orlando , FL , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Central Florida Orlando , FL , USA","institution_ids":["https://openalex.org/I106165777"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000183128","display_name":"El Houcine Bergou","orcid":"https://orcid.org/0000-0001-8685-6974","observed_orcids":["https://orcid.org/0000-0001-8685-6974"]},"institutions":[{"id":"https://openalex.org/I4210131560","display_name":"Universit\u00e9 Mohammed VI Polytechnique","ror":"https://ror.org/03xc55g68","country_code":"MA","type":"education","lineage":["https://openalex.org/I4210131560"]}],"countries":["MA"],"is_corresponding":false,"raw_author_name":"Bergou, El Houcine","raw_affiliation_strings":["Mohammed VI Polytechnic University Ben Guerir , Morocco"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohammed VI Polytechnic University Ben Guerir , Morocco","institution_ids":["https://openalex.org/I4210131560"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074006719","display_name":"Aritra Dutta","orcid":"https://orcid.org/0000-0001-6994-1659","observed_orcids":["https://orcid.org/0000-0001-6994-1659"]},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dutta, Aritra","raw_affiliation_strings":["University of Central Florida Orlando , FL , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Central Florida Orlando , FL , USA","institution_ids":["https://openalex.org/I106165777"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10963","display_name":"Advanced Optimization Algorithms Research","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/2612","display_name":"Numerical Analysis"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/derivative-free-optimization","display_name":"derivative-free optimization","score":0.9700000286102295},{"id":"https://openalex.org/keywords/first-order-methods","display_name":"first-order methods","score":0.9350000023841858},{"id":"https://openalex.org/keywords/step-size-selection","display_name":"step size selection","score":0.8489999771118164},{"id":"https://openalex.org/keywords/llm-fine-tuning","display_name":"LLM fine-tuning","score":0.7730000019073486},{"id":"https://openalex.org/keywords/line-search","display_name":"line search","score":0.5820000171661377},{"id":"https://openalex.org/keywords/convergence-guarantees","display_name":"convergence guarantees","score":0.6930000185966492},{"id":"https://openalex.org/keywords/critical-points","display_name":"critical points","score":0.4320000112056732},{"id":"https://openalex.org/keywords/neural-network-optimization","display_name":"neural network optimization","score":0.5559999942779541}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6886000037193298},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.6079000234603882},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.6011999845504761},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5303000211715698},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.49230000376701355},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4823000133037567},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.4650999903678894},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4560000002384186},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42489999532699585},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4056999981403351},{"id":"https://openalex.org/C198352243","wikidata":"https://www.wikidata.org/wiki/Q37105","display_name":"Line (geometry)","level":2,"score":0.3758000135421753},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3434000015258789},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3425000011920929},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3411000072956085},{"id":"https://openalex.org/C195065555","wikidata":"https://www.wikidata.org/wiki/Q214881","display_name":"Curvature","level":2,"score":0.335099995136261},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.3310999870300293},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33070001006126404},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.3018999993801117},{"id":"https://openalex.org/C141934464","wikidata":"https://www.wikidata.org/wiki/Q3305386","display_name":"Local optimum","level":2,"score":0.29789999127388},{"id":"https://openalex.org/C135320971","wikidata":"https://www.wikidata.org/wiki/Q1868524","display_name":"Local search (optimization)","level":2,"score":0.28929999470710754},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26030001044273376},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.2551000118255615}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.02190","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02190","pdf_url":"https://arxiv.org/pdf/2610.02190","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.02190","doi":"https://doi.org/10.48550/arxiv.2610.02190","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.02190","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.02190","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02190","pdf_url":"https://arxiv.org/pdf/2610.02190","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2169903659","display_name":"Scholarships, Academic, and Social Supports to Provide Low-Income Transfers Students Opportunities for Nurtured Growth in AI","funder_award_id":"2321986","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309360","display_name":"Florida Department of Health","ror":"https://ror.org/03e1xkn26"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7216515338.pdf","grobid_xml":"https://content.openalex.org/works/W7216515338.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W2622263826","https://openalex.org/W2736601468","https://openalex.org/W2794325560","https://openalex.org/W2795247881","https://openalex.org/W3210904070","https://openalex.org/W4391631327","https://openalex.org/W4399116024","https://openalex.org/W4401306886","https://openalex.org/W4401306979","https://openalex.org/W4415132714","https://openalex.org/W4415311914","https://openalex.org/W4416417373","https://openalex.org/W4417113061","https://openalex.org/W6903592333"],"related_works":["https://openalex.org/W7129243956","https://openalex.org/W7140188843","https://openalex.org/W7103987430","https://openalex.org/W4416184865","https://openalex.org/W7166867903","https://openalex.org/W7119609575","https://openalex.org/W7118489425","https://openalex.org/W4392182205","https://openalex.org/W7166837375","https://openalex.org/W7221769395"],"abstract_inverted_index":{"Step-size":[0],"selection":[1,33],"remains":[2],"a":[3,27,38,75,88,92,106,130,142,145],"central":[4],"challenge":[5],"in":[6],"large-scale":[7],"neural":[8],"network":[9],"optimization;":[10],"conservative":[11],"steps":[12,17],"slow":[13],"convergence,":[14],"while":[15],"aggressive":[16],"can":[18],"destabilize":[19],"it.":[20],"We":[21,110],"combine":[22],"\\textbf{Z}ero-and-\\textbf{F}irst-\\textbf{O}rder":[23],"optimization~(ZFO)":[24],"and":[25,46,66,86,137,154,160,171],"propose":[26],"lightweight":[28],"framework":[29],"that":[30,102,116,124,138],"decouples":[31],"direction":[32,45,85],"from":[34],"step-size.":[35],"ZFO":[36,72,139,156],"uses":[37],"trusted":[39],"first-order":[40,166],"optimizer":[41],"to":[42,55,59,114,141,164],"determine":[43],"the":[44,62,79,83,125,134,149,169,177],"performs":[47],"zeroth-order":[48],"evaluations":[49,118],"only":[50],"along":[51,82,133],"this":[52],"one-dimensional":[53],"subspace":[54],"choose":[56],"how":[57],"far":[58],"move.":[60],"Using":[61],"current":[63],"{gradient":[64],"information}":[65],"two":[67],"additional":[68],"objective":[69,80],"function":[70,81],"evaluations,":[71],"instances":[73],"construct":[74],"local":[76,127,173],"model":[77,128,174],"of":[78,144],"proposed":[84],"select":[87],"curvature-aware":[89],"step":[90,132],"within":[91],"bounded":[93],"search":[94,135],"interval.":[95],"This":[96],"yields":[97],"an":[98],"adaptive":[99],"step-selection":[100],"mechanism":[101],"costs":[103],"less":[104],"than":[105],"full":[107],"line":[108],"search.":[109],"provide":[111],"theoretical":[112],"guarantees":[113],"show":[115],"shared-sample":[117],"produce":[119],"reliable":[120],"finite-difference":[121],"curvature":[122],"estimates,":[123],"induced":[126],"selects":[129],"near-optimal":[131],"interval,":[136],"converges":[140],"neighborhood":[143],"stationary":[146],"point.":[147],"Across":[148],"evaluated":[150],"settings,":[151],"language":[152],"models":[153],"datasets,":[155],"frequently":[157],"improves":[158],"optimization":[159],"final":[161],"performance":[162],"relative":[163],"fixed-step":[165],"baselines,":[167],"with":[168],"magnitude":[170],"preferred":[172],"depending":[175],"on":[176],"objective.":[178],"Our":[179],"code":[180],"is":[181],"publicly":[182],"available":[183],"at:":[184],"https://github.com/nizswan/Zeroth-First-Order-Framework.":[185]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-03T00:00:00"}