{"id":"https://openalex.org/W7138097978","doi":"https://doi.org/10.1609/aaai.v40i33.40062","title":"SALR: Sparsity-Aware Low-Rank Representation for Efficient Fine-Tuning of Large Language Models","display_name":"SALR: Sparsity-Aware Low-Rank Representation for Efficient Fine-Tuning of Large Language Models","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138097978","doi":"https://doi.org/10.1609/aaai.v40i33.40062"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i33.40062","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i33.40062","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40062/44023","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40062/44023","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129669191","display_name":"Longteng Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Longteng Zhang","raw_affiliation_strings":["The Hong Kong University of Science and Technology (Guangzhou)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology (Guangzhou)","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123890279","display_name":"Sen Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sen Wu","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129657944","display_name":"Shuai Hou","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuai Hou","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123962987","display_name":"Zhengyu Qing","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhengyu Qing","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129742415","display_name":"Zhuo Zheng","orcid":null},"institutions":[{"id":"https://openalex.org/I4210160618","display_name":"Huawei Technologies (United Kingdom)","ror":"https://ror.org/056gzgs71","country_code":"GB","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210160618"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhuo Zheng","raw_affiliation_strings":["Huawei Technologies"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies","institution_ids":["https://openalex.org/I4210160618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129644361","display_name":"Danning Ke","orcid":null},"institutions":[{"id":"https://openalex.org/I4210160618","display_name":"Huawei Technologies (United Kingdom)","ror":"https://ror.org/056gzgs71","country_code":"GB","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210160618"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Danning Ke","raw_affiliation_strings":["Huawei Technologies"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies","institution_ids":["https://openalex.org/I4210160618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124035910","display_name":"Qihong Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I4210160618","display_name":"Huawei Technologies (United Kingdom)","ror":"https://ror.org/056gzgs71","country_code":"GB","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210160618"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Qihong Lin","raw_affiliation_strings":["Huawei Technologies"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies","institution_ids":["https://openalex.org/I4210160618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129677129","display_name":"Qiang Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiang Wang","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129686256","display_name":"Shaohuai Shi","orcid":null},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaohuai Shi","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5129645934","display_name":"Xiaowen Chu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaowen Chu","raw_affiliation_strings":["The Hong Kong University of Science and Technology (Guangzhou)\nThe Hong Kong University of Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology (Guangzhou)\nThe Hong Kong University of Science and Technology","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"33","first_page":"28337","last_page":"28345"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.3091000020503998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.3091000020503998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.09910000115633011,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.09459999948740005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.7967000007629395},{"id":"https://openalex.org/keywords/fuse","display_name":"Fuse (electrical)","score":0.5881999731063843},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.5810999870300293},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5706999897956848},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.5077999830245972},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.503600001335144},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4851999878883362},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.461899995803833},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.4207000136375427}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.7967000007629395},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7688000202178955},{"id":"https://openalex.org/C141353440","wikidata":"https://www.wikidata.org/wiki/Q182221","display_name":"Fuse (electrical)","level":2,"score":0.5881999731063843},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.5810999870300293},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5706999897956848},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.5077999830245972},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.503600001335144},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4851999878883362},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4722000062465668},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.461899995803833},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.4207000136375427},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4052000045776367},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.3720000088214874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3662000000476837},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.36149999499320984},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.3549000024795532},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3463999927043915},{"id":"https://openalex.org/C2779231336","wikidata":"https://www.wikidata.org/wiki/Q7534724","display_name":"Sketch","level":2,"score":0.34540000557899475},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.32749998569488525},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.30559998750686646},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.30160000920295715},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28859999775886536},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.2840000092983246},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C111696304","wikidata":"https://www.wikidata.org/wiki/Q2303697","display_name":"Sorting","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C34146451","wikidata":"https://www.wikidata.org/wiki/Q5048094","display_name":"Cascade","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2630000114440918}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1609/aaai.v40i33.40062","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i33.40062","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40062/44023","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2601.16991","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2601.16991","pdf_url":"https://arxiv.org/pdf/2601.16991","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/40062","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/40062","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i33.40062","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i33.40062","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40062/44023","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7138097978.pdf","grobid_xml":"https://content.openalex.org/works/W7138097978.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Adapting":[0],"large":[1],"pre-trained":[2],"language":[3],"models":[4,54],"to":[5,156,189],"downstream":[6],"tasks":[7],"often":[8],"entails":[9],"fine-tuning":[10,74],"millions":[11],"of":[12,123,175],"parameters":[13,32],"or":[14],"deploying":[15],"costly":[16],"dense":[17,40],"weight":[18,35],"updates,":[19,36],"which":[20,115],"hinders":[21],"their":[22],"use":[23],"in":[24],"resource-constrained":[25],"environments.":[26],"Low-rank":[27],"Adaptation":[28],"(LoRA)":[29],"reduces":[30,117,181],"trainable":[31],"by":[33,120,184],"factorizing":[34],"yet":[37],"the":[38,94,99,106,173],"underlying":[39],"weights":[41,97],"still":[42],"impose":[43],"high":[44],"storage":[45],"and":[46,103,142,161,179,186],"computation":[47],"costs.":[48],"Magnitude-based":[49],"pruning":[50,82,92,100],"can":[51],"yield":[52],"sparse":[53,81],"but":[55],"typically":[56],"degrades":[57],"LoRA\u2019s":[58],"performance":[59,174],"when":[60],"applied":[61],"naively.":[62],"In":[63],"this":[64],"paper,":[65],"we":[66,104,132,143],"introduce":[67],"SALR":[68,164],"(Sparsity-Aware":[69],"Low-Rank":[70],"Representation),":[71],"a":[72,84,111,121,138,145,149,190],"novel":[73],"paradigm":[75],"that":[76,90],"unifies":[77],"low-rank":[78,113,135],"adaptation":[79],"with":[80,148],"under":[83],"rigorous":[85],"mean-squared-error":[86],"framework.":[87],"We":[88],"prove":[89],"statically":[91],"only":[93],"frozen":[95],"base":[96],"minimizes":[98],"error":[101],"bound,":[102],"recover":[105],"discarded":[107],"residual":[108],"information":[109],"via":[110],"truncated-SVD":[112],"adapter,":[114],"provably":[116],"per-entry":[118],"MSE":[119],"factor":[122],"(1":[124],"-":[125],"r/min(d,":[126],"k)).":[127],"To":[128],"maximize":[129],"hardware":[130],"efficiency,":[131],"fuse":[133],"multiple":[134],"adapters":[136],"into":[137],"single":[139],"concatenated":[140],"GEMM,":[141],"adopt":[144],"bitmap-based":[146],"encoding":[147],"two-stage":[150],"pipelined":[151],"decoding":[152],"+":[153],"GEMM":[154],"design":[155],"achieve":[157],"true":[158],"model":[159,182],"compression":[160],"speedup.":[162,193],"Empirically,":[163],"attains":[165],"50%":[166],"sparsity":[167],"on":[168,177],"various":[169],"LLMs":[170],"while":[171],"matching":[172],"LoRA":[176],"GSM8K":[178],"MMLU,":[180],"size":[183],"2x,":[185],"delivers":[187],"up":[188],"1.7x":[191],"inference":[192]},"counts_by_year":[],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2026-03-18T00:00:00"}