{"id":"https://openalex.org/W4385848613","doi":"https://doi.org/10.48550/arxiv.2308.06522","title":"SLoRA: Federated Parameter Efficient Fine-Tuning of Language Models","display_name":"SLoRA: Federated Parameter Efficient Fine-Tuning of Language Models","publication_year":2023,"publication_date":"2023-08-12","ids":{"openalex":"https://openalex.org/W4385848613","doi":"https://doi.org/10.48550/arxiv.2308.06522"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2308.06522","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.06522","pdf_url":"https://arxiv.org/pdf/2308.06522","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2308.06522","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061117202","display_name":"Sara Babakniya","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Babakniya, Sara","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083855675","display_name":"Ahmed Roushdy Elkordy","orcid":"https://orcid.org/0000-0002-6090-1789"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elkordy, Ahmed Roushdy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005778563","display_name":"Yahya H. Ezzeldin","orcid":"https://orcid.org/0000-0002-4238-5362"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ezzeldin, Yahya H.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100349220","display_name":"Qingfeng Liu","orcid":"https://orcid.org/0000-0001-8857-6677"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Qingfeng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113919277","display_name":"Kee-Bong Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Kee-Bong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022805456","display_name":"Mostafa El\u2010Khamy","orcid":"https://orcid.org/0000-0001-9421-6037"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"El-Khamy, Mostafa","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112726818","display_name":"Salman Avestimehr","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Avestimehr, Salman","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5061117202"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9527999758720398,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8199560046195984},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.7266979217529297},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6853622198104858},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"Fine-tuning","score":0.6566907167434692},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.571128785610199},{"id":"https://openalex.org/keywords/federated-learning","display_name":"Federated learning","score":0.5695627331733704},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5574166774749756},{"id":"https://openalex.org/keywords/edge-device","display_name":"Edge device","score":0.5366747975349426},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.5257459878921509},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4890028238296509},{"id":"https://openalex.org/keywords/enhanced-data-rates-for-gsm-evolution","display_name":"Enhanced Data Rates for GSM Evolution","score":0.4818904995918274},{"id":"https://openalex.org/keywords/performance-tuning","display_name":"Performance tuning","score":0.4650740623474121},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.43900808691978455},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"Synthetic data","score":0.4214383363723755},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36358821392059326},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3567540943622589},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3549390435218811},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.12201762199401855}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8199560046195984},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7266979217529297},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6853622198104858},{"id":"https://openalex.org/C157524613","wikidata":"https://www.wikidata.org/wiki/Q2828883","display_name":"Fine-tuning","level":2,"score":0.6566907167434692},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.571128785610199},{"id":"https://openalex.org/C2992525071","wikidata":"https://www.wikidata.org/wiki/Q50818671","display_name":"Federated learning","level":2,"score":0.5695627331733704},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5574166774749756},{"id":"https://openalex.org/C138236772","wikidata":"https://www.wikidata.org/wiki/Q25098575","display_name":"Edge device","level":3,"score":0.5366747975349426},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.5257459878921509},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4890028238296509},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.4818904995918274},{"id":"https://openalex.org/C2777138346","wikidata":"https://www.wikidata.org/wiki/Q1714153","display_name":"Performance tuning","level":2,"score":0.4650740623474121},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.43900808691978455},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.4214383363723755},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36358821392059326},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3567540943622589},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3549390435218811},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.12201762199401855},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2308.06522","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.06522","pdf_url":"https://arxiv.org/pdf/2308.06522","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2308.06522","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.06522","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2308.06522","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.06522","pdf_url":"https://arxiv.org/pdf/2308.06522","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4385848613.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4298221930","https://openalex.org/W2777914285","https://openalex.org/W3013363440","https://openalex.org/W4287823391","https://openalex.org/W4320067866","https://openalex.org/W4306353077","https://openalex.org/W4312868944","https://openalex.org/W4385848613","https://openalex.org/W4288099469","https://openalex.org/W3164683393"],"abstract_inverted_index":{"Transfer":[0],"learning":[1],"via":[2],"fine-tuning":[3,64,84,111],"pre-trained":[4],"transformer":[5,61],"models":[6],"has":[7],"gained":[8],"significant":[9,161],"success":[10],"in":[11,87,137],"delivering":[12],"state-of-the-art":[13],"results":[14,150],"across":[15,102],"various":[16],"NLP":[17],"tasks.":[18,93],"In":[19],"the":[20,36,45,56,75,100,107,112,132],"absence":[21],"of":[22,35,52,59,135],"centralized":[23],"data,":[24],"Federated":[25],"Learning":[26],"(FL)":[27],"can":[28],"benefit":[29],"from":[30],"distributed":[31],"and":[32,49,55,77,114],"private":[33],"data":[34,101,140],"FL":[37,89],"edge":[38,53],"clients":[39],"for":[40,91],"fine-tuning.":[41],"However,":[42],"due":[43],"to":[44,67,157,175],"limited":[46],"communication,":[47],"computation,":[48],"storage":[50],"capabilities":[51],"devices":[54],"huge":[57],"sizes":[58],"popular":[60],"models,":[62],"efficient":[63,83],"is":[65],"crucial":[66],"make":[68],"federated":[69],"training":[70,171],"feasible.":[71],"This":[72],"work":[73],"explores":[74],"opportunities":[76],"challenges":[78],"associated":[79],"with":[80,160,164],"applying":[81],"parameter":[82],"(PEFT)":[85],"methods":[86,117],"different":[88],"settings":[90],"language":[92],"Specifically,":[94],"our":[95],"investigation":[96],"reveals":[97],"that":[98,152],"as":[99],"users":[103],"becomes":[104],"more":[105],"diverse,":[106],"gap":[108],"between":[109],"fully":[110],"model":[113],"employing":[115],"PEFT":[116],"widens.":[118],"To":[119],"bridge":[120],"this":[121],"performance":[122,155],"gap,":[123],"we":[124],"propose":[125],"a":[126,143],"method":[127],"called":[128],"SLoRA,":[129],"which":[130],"overcomes":[131],"key":[133],"limitations":[134],"LoRA":[136],"high":[138],"heterogeneous":[139],"scenarios":[141],"through":[142],"novel":[144],"data-driven":[145],"initialization":[146],"technique.":[147],"Our":[148],"experimental":[149],"demonstrate":[151],"SLoRA":[153],"achieves":[154],"comparable":[156],"full":[158],"fine-tuning,":[159],"sparse":[162],"updates":[163],"approximately":[165],"$\\sim":[166],"1\\%$":[167],"density":[168],"while":[169],"reducing":[170],"time":[172],"by":[173],"up":[174],"$90\\%$.":[176]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}