{"id":"https://openalex.org/W7163665632","doi":"https://doi.org/10.48550/arxiv.2606.05516","title":"Dominant-Layer ZO: A Single Layer Dominates Zeroth-Order Fine-Tuning of LLMs","display_name":"Dominant-Layer ZO: A Single Layer Dominates Zeroth-Order Fine-Tuning of LLMs","publication_year":2026,"publication_date":"2026-06-03","ids":{"openalex":"https://openalex.org/W7163665632","doi":"https://doi.org/10.48550/arxiv.2606.05516"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.05516","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.05516","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.05516","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109748593","display_name":"Wanhao Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Wanhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137967671","display_name":"Ziyan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Ziyan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137950200","display_name":"Zheng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138004186","display_name":"Abeer Matar Almalky","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Almalky, Abeer Matar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137996548","display_name":"Yihang Zuo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zuo, Yihang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028748926","display_name":"Shuteng Niu","orcid":"https://orcid.org/0000-0002-1069-9236"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Niu, Shuteng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137949284","display_name":"Sen Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Sen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138014563","display_name":"Adnan Siraj Rakin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rakin, Adnan Siraj","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137924701","display_name":"Deliang Fan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan, Deliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137946474","display_name":"Li Yang (6520)","orcid":"https://orcid.org/0009-0000-6793-7840"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18039999902248383,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18039999902248383,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.09070000052452087,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.09030000120401382,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.5450000166893005},{"id":"https://openalex.org/keywords/perturbation","display_name":"Perturbation (astronomy)","score":0.5271999835968018},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.5130000114440918},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.5116000175476074},{"id":"https://openalex.org/keywords/physical-layer","display_name":"Physical layer","score":0.335099995136261}],"concepts":[{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.5450000166893005},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.5271999835968018},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.5130000114440918},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.5116000175476074},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42080000042915344},{"id":"https://openalex.org/C19247436","wikidata":"https://www.wikidata.org/wiki/Q192727","display_name":"Physical layer","level":3,"score":0.335099995136261},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.2964000105857849},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.2919999957084656},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2628999948501587},{"id":"https://openalex.org/C16895185","wikidata":"https://www.wikidata.org/wiki/Q15101726","display_name":"Perfectly matched layer","level":3,"score":0.24140000343322754}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.05516","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.05516","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.05516","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.05516","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Zeroth-order":[0],"(ZO)":[1],"optimization":[2,159],"enables":[3],"memory-efficient":[4],"fine-tuning":[5,35,51,177,187],"of":[6,86],"large":[7],"language":[8],"models":[9],"(LLMs)":[10],"using":[11],"only":[12],"forward":[13],"passes,":[14],"but":[15,73],"it":[16],"remains":[17],"unclear":[18],"how":[19,110],"useful":[20],"adaptation":[21],"is":[22,36,71],"distributed":[23],"across":[24,170],"layers.":[25,148],"In":[26],"this":[27,52,106,152],"work,":[28],"we":[29,108],"reveal":[30],"a":[31,40,82,150],"surprising":[32],"phenomenon:":[33],"ZO":[34,62,115,176,186],"sharply":[37],"dominated":[38],"by":[39],"single":[41],"decoding":[42,147],"layer.":[43],"Across":[44],"multiple":[45],"LLM":[46],"families":[47],"and":[48,75,130,142,157,168,184],"downstream":[49],"tasks,":[50],"dominant":[53,69,91,121],"layer":[54,70,92,99,122,153],"alone":[55],"consistently":[56,93],"matches":[57],"or":[58],"even":[59],"exceeds":[60],"full-model":[61,182],"fine-tuning.":[63],"We":[64,117],"further":[65],"show":[66,173],"that":[67,119,174],"the":[68,90,96,101,120,134],"task-agnostic":[72],"model-specific,":[74],"can":[76],"be":[77],"identified":[78],"before":[79],"training":[80,193],"through":[81,144],"simple":[83],"inference-only":[84],"analysis":[85],"activation":[87],"outliers.":[88],"Specifically,":[89],"aligns":[94],"with":[95],"first":[97],"activation-outlier":[98],"in":[100,133],"pre-trained":[102],"model.":[103],"To":[104],"explain":[105],"phenomenon,":[107],"analyze":[109],"perturbation":[111,128],"effects":[112,139],"propagate":[113,141],"under":[114,161],"optimization.":[116],"find":[118],"combines":[123],"two":[124],"key":[125],"properties:":[126],"high":[127],"sensitivity":[129],"early":[131],"placement":[132],"residual":[135],"stream,":[136],"allowing":[137],"perturbation-induced":[138],"to":[140,191],"accumulate":[143],"remaining":[145],"subsequent":[146],"As":[149],"result,":[151],"produces":[154],"disproportionately":[155],"strong":[156],"stable":[158],"signals":[160],"forward-only":[162],"updates.":[163],"Extensive":[164],"experiments":[165],"on":[166],"LLaMA2-7B":[167],"Qwen3-8B":[169],"nine":[171],"benchmarks":[172],"dominant-layer":[175],"improves":[178],"average":[179],"performance":[180],"over":[181],"MeZO":[183],"LoRA-based":[185],"while":[188],"achieving":[189],"up":[190],"4.52$\\times$":[192],"speedup.":[194]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-06T00:00:00"}