{"id":"https://openalex.org/W4409347839","doi":"https://doi.org/10.1609/aaai.v39i23.34587","title":"Gradient Weight-normalized Low-rank Projection for Efficient LLM Training","display_name":"Gradient Weight-normalized Low-rank Projection for Efficient LLM Training","publication_year":2025,"publication_date":"2025-04-11","ids":{"openalex":"https://openalex.org/W4409347839","doi":"https://doi.org/10.1609/aaai.v39i23.34587"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v39i23.34587","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i23.34587","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34587/36742","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34587/36742","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091066062","display_name":"Jia-Hong Huang","orcid":"https://orcid.org/0000-0001-7943-2591"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jia-Hong Huang","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045786583","display_name":"Yixian Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Yixian Shen","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044091982","display_name":"Hongyi Zhu","orcid":"https://orcid.org/0009-0006-0298-0905"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Hongyi Zhu","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075331928","display_name":"Stevan Rudinac","orcid":"https://orcid.org/0000-0003-1904-8736"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Stevan Rudinac","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055639036","display_name":"Evangelos Kanoulas","orcid":"https://orcid.org/0000-0002-8312-0694"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Evangelos Kanoulas","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":11.0132,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.98361538,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"39","issue":"23","first_page":"24123","last_page":"24131"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14158","display_name":"Optical Systems and Laser Technology","score":0.9702000021934509,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14158","display_name":"Optical Systems and Laser Technology","score":0.9702000021934509,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9555000066757202,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10191","display_name":"Robotics and Sensor-Based Localization","score":0.9541000127792358,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.6556276082992554},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6454750299453735},{"id":"https://openalex.org/keywords/projection","display_name":"Projection (relational algebra)","score":0.5656682848930359},{"id":"https://openalex.org/keywords/proximal-gradient-methods","display_name":"Proximal Gradient Methods","score":0.4775213301181793},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.4310171604156494},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.378188818693161},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.35686156153678894},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.32694894075393677},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.2080100178718567},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.17978990077972412},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.14186552166938782},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.10507205128669739},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.07702460885047913}],"concepts":[{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6556276082992554},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6454750299453735},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.5656682848930359},{"id":"https://openalex.org/C10494615","wikidata":"https://www.wikidata.org/wiki/Q17086765","display_name":"Proximal Gradient Methods","level":4,"score":0.4775213301181793},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4310171604156494},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.378188818693161},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.35686156153678894},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.32694894075393677},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.2080100178718567},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.17978990077972412},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.14186552166938782},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.10507205128669739},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.07702460885047913},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1609/aaai.v39i23.34587","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i23.34587","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34587/36742","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:dare.uva.nl:publications/cc10b17f-947a-40ab-a428-91029eb657e0","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/gradient-weightnormalized-lowrank-projection-for-efficient-llm-training(cc10b17f-947a-40ab-a428-91029eb657e0).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Huang, J-H, Shen, Y, Zhu, H, Rudinac, S & Kanoulas, E 2025, Gradient Weight-normalized Low-rank Projection for Efficient LLM Training. in T Walsh, J Shah & Z Kolter (eds), Proceedings of the 39th Annual AAAI Conference on Artificial Intelligence : February 25-March 4, 2025, Philadelphia, Pennsylvania, USA. vol. 23, Washington, DC, pp. 24123-24131, 39th Annual AAAI Conference on Artificial Intelligence, Philadelphia, Pennsylvania, United States, 25/02/25. https://doi.org/10.1609/aaai.v39i23.34587","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:dare.uva.nl:openaire/cc10b17f-947a-40ab-a428-91029eb657e0","is_oa":true,"landing_page_url":"https://hdl.handle.net/11245.1/cc10b17f-947a-40ab-a428-91029eb657e0","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Huang, J-H, Shen, Y, Zhu, H, Rudinac, S & Kanoulas, E 2025, Gradient Weight-normalized Low-rank Projection for Efficient LLM Training. in T Walsh, J Shah & Z Kolter (eds), Proceedings of the 39th Annual AAAI Conference on Artificial Intelligence : February 25-March 4, 2025, Philadelphia, Pennsylvania, USA. vol. 23, Washington, DC, pp. 24123-24131, 39th Annual AAAI Conference on Artificial Intelligence, Philadelphia, Pennsylvania, United States, 25/02/25. https://doi.org/10.1609/aaai.v39i23.34587","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1609/aaai.v39i23.34587","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i23.34587","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34587/36742","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409347839.pdf","grobid_xml":"https://content.openalex.org/works/W4409347839.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2564735875","https://openalex.org/W230091440","https://openalex.org/W2963086517","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W3014540454"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4,39],"shown":[5],"remarkable":[6],"performance":[7,78],"across":[8],"various":[9],"tasks,":[10],"but":[11,42],"the":[12,23,84,102,130,143,164],"escalating":[13],"demands":[14],"on":[15,139,167],"computational":[16],"resources":[17],"pose":[18],"significant":[19],"challenges,":[20],"particularly":[21],"in":[22,157],"extensive":[24],"utilization":[25],"of":[26,132,174,181,186],"full":[27,48,80],"fine-tuning":[28,36,49,158,163],"for":[29,196],"downstream":[30],"tasks.":[31,159],"To":[32],"address":[33],"this,":[34],"parameter-efficient":[35],"(PEFT)":[37],"methods":[38,156],"been":[40],"developed,":[41],"they":[43],"often":[44],"underperform":[45],"compared":[46],"to":[47,79,87,101,126],"and":[50,72,104,128,200],"struggle":[51],"with":[52,171],"memory":[53,73,109,122],"efficiency.":[54],"In":[55],"this":[56],"work,":[57],"we":[58],"introduce":[59],"Gradient":[60],"Weight-Normalized":[61],"Low-Rank":[62],"Projection":[63],"(GradNormLoRP),":[64],"a":[65,172,193],"novel":[66],"approach":[67],"that":[68,116],"enhances":[69],"both":[70],"parameter":[71],"efficiency":[74],"while":[75],"maintaining":[76],"comparable":[77],"fine-tuning.":[81,201],"GradNormLoRP":[82,119,152,176,191],"normalizes":[83],"weight":[85,103],"matrix":[86],"improve":[88],"gradient":[89,105],"conditioning,":[90],"facilitating":[91],"better":[92],"convergence":[93],"during":[94,111],"optimization.":[95],"Additionally,":[96],"it":[97],"applies":[98],"low-rank":[99,155],"approximations":[100],"matrices,":[106],"significantly":[107],"reducing":[108],"usage":[110,123],"training.":[112],"Extensive":[113],"experiments":[114],"demonstrate":[115],"our":[117],"8-bit":[118],"reduces":[120],"optimizer":[121],"by":[124],"up":[125],"89.5\\%":[127],"enables":[129],"pre-training":[131,199],"large":[133],"LLMs,":[134],"such":[135],"as":[136,192],"LLaMA":[137],"7B,":[138],"consumer-level":[140],"GPUs":[141],"like":[142],"NVIDIA":[144],"RTX":[145],"4090,":[146],"without":[147],"additional":[148],"inference":[149],"costs.":[150],"Moreover,":[151],"outperforms":[153],"existing":[154],"For":[160],"instance,":[161],"when":[162],"RoBERTa":[165],"model":[166],"all":[168],"GLUE":[169],"tasks":[170],"rank":[173],"8,":[175],"achieves":[177],"an":[178],"average":[179],"score":[180,185],"80.65,":[182],"surpassing":[183],"LoRA's":[184],"79.23.":[187],"These":[188],"results":[189],"underscore":[190],"promising":[194],"alternative":[195],"efficient":[197],"LLM":[198]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}