{"id":"https://openalex.org/W4416034066","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.676","title":"LoRaDA: Low-Rank Direct Attention Adaptation for Efficient LLM Fine-tuning","display_name":"LoRaDA: Low-Rank Direct Attention Adaptation for Efficient LLM Fine-tuning","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416034066","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.676"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.676","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.676","pdf_url":"https://aclanthology.org/2025.findings-emnlp.676.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-emnlp.676.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088925790","display_name":"Zhangming Li","orcid":"https://orcid.org/0009-0000-7097-7667"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhangming Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102016069","display_name":"Qinghao Hu","orcid":"https://orcid.org/0009-0007-5300-5787"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qinghao Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100674350","display_name":"Yiqun Chen","orcid":"https://orcid.org/0000-0003-4218-084X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yiqun Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079244166","display_name":"Peisong Wang","orcid":"https://orcid.org/0000-0002-6384-0280"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peisong Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056548830","display_name":"Yifan Zhang","orcid":"https://orcid.org/0000-0002-9182-6267"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yifan Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112149353","display_name":"Jian Cheng","orcid":"https://orcid.org/0000-0003-1289-2758"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jian Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.40099999,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"12638","last_page":"12655"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12079","display_name":"IoT Networks and Protocols","score":0.3855000138282776,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12079","display_name":"IoT Networks and Protocols","score":0.3855000138282776,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13553","display_name":"Age of Information Optimization","score":0.11959999799728394,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.08190000057220459,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adaptation","display_name":"Adaptation (eye)","score":0.33500000834465027},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.2685999870300293},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.26570001244544983},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.2655999958515167},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.24560000002384186}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5537999868392944},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40619999170303345},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.33500000834465027},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.24560000002384186},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.24449999630451202},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.23819999396800995},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.23469999432563782}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-emnlp.676","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.676","pdf_url":"https://aclanthology.org/2025.findings-emnlp.676.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.676","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.676","pdf_url":"https://aclanthology.org/2025.findings-emnlp.676.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3352541487","display_name":null,"funder_award_id":"BK20243051","funder_id":"https://openalex.org/F4320322769","funder_display_name":"Natural Science Foundation of Jiangsu Province"}],"funders":[{"id":"https://openalex.org/F4320321605","display_name":"Government of Jiangsu Province","ror":"https://ror.org/004svx814"},{"id":"https://openalex.org/F4320322769","display_name":"Natural Science Foundation of Jiangsu Province","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416034066.pdf","grobid_xml":"https://content.openalex.org/works/W4416034066.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0,51],"the":[1,35,47,52,124,158,168,183],"parameter":[2],"size":[3],"of":[4,34,37,41,126,161,185],"language":[5],"models":[6],"becomes":[7],"extremely":[8],"large,":[9],"fine-tuning":[10,23,79,170],"them":[11],"with":[12,147,191],"limited":[13],"resources":[14],"has":[15],"become":[16],"a":[17,31,88,101,133,179],"challenging":[18],"task.Latest":[19],"advancements":[20],"in":[21,68],"parameterefficient":[22],"(PEFT)":[24],"techniques":[25],"allow":[26],"for":[27,95],"adjustments":[28],"to":[29,114,135,174,189],"only":[30],"minor":[32],"fraction":[33],"parameters":[36],"these":[38,75,84],"LLMs.Yet,":[39],"most":[40],"PEFT":[42,56],"methods":[43,57,76],"may":[44],"suffer":[45],"from":[46],"following":[48],"limitations:":[49],"(1)":[50],"rank":[53,150],"decreases":[54],"sharply,":[55],"like":[58],"LoRA":[59,140,186],"and":[60,77,117,141],"Adapter":[61],"tuning":[62],"will":[63],"exhibit":[64],"significant":[65],"performance":[66,145,160],"degradation":[67],"downstream":[69,127,155],"tasks.(2)":[70],"An":[71],"accuracy":[72,184],"gap":[73],"between":[74],"full":[78,169],"(Full-FT)":[80],"still":[81],"exists.To":[82],"tackle":[83],"problems,":[85],"we":[86,99],"propose":[87],"Low-Rank":[89],"Direct":[90],"Attention":[91,105],"Adaptation":[92],"(LoRaDA)":[93],"method":[94,171],"efficient":[96],"LLM":[97],"fine-tuning.Specifically,":[98],"introduce":[100],"novel":[102],"Low-rank":[103],"Multi-head":[104],"Map":[106],"Module":[107],"(LMAM),":[108],"which":[109],"can":[110,130],"bring":[111],"negative":[112],"attention":[113,120],"self-attention":[115],"modules":[116],"learn":[118],"low-rank":[119],"weights":[121],"directly,":[122],"capturing":[123],"characteristics":[125],"tasks.Furthermore,":[128],"LMAM":[129,181],"serve":[131],"as":[132,139],"plug-in":[134],"existing":[136],"methods,":[137],"such":[138],"Adapter,":[142],"providing":[143],"state-of-theart":[144],"even":[146,166],"extreme":[148],"low":[149],"setting.Extensive":[151],"experiments":[152],"on":[153,176,193],"various":[154],"tasks":[156],"demonstrate":[157],"superior":[159],"our":[162],"LoRaDA":[163,165],"method.Specifically,":[164],"outperforms":[167],"by":[172,187],"up":[173,188],"2.1%":[175],"GLUE":[177],"benchmark.As":[178],"plug-in,":[180],"boosts":[182],"27.7%":[190],"LLaMA-7B":[192],"Commonsense":[194],"Reasoning":[195],"benchmark.":[196]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-11-08T00:00:00"}