{"id":"https://openalex.org/W4406611429","doi":"https://doi.org/10.1109/smc54092.2024.10831406","title":"Efficient One-Shot Pruning of Large Language Models with Low-Rank Approximation","display_name":"Efficient One-Shot Pruning of Large Language Models with Low-Rank Approximation","publication_year":2024,"publication_date":"2024-10-06","ids":{"openalex":"https://openalex.org/W4406611429","doi":"https://doi.org/10.1109/smc54092.2024.10831406"},"language":"en","primary_location":{"id":"doi:10.1109/smc54092.2024.10831406","is_oa":false,"landing_page_url":"https://doi.org/10.1109/smc54092.2024.10831406","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Systems, Man, and Cybernetics (SMC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101038839","display_name":"Yangyan Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yangyan Xu","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102823047","display_name":"Cong Cao","orcid":"https://orcid.org/0000-0002-6853-6421"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cong Cao","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102878238","display_name":"Fangfang Yuan","orcid":"https://orcid.org/0000-0002-1255-1999"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fangfang Yuan","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093910669","display_name":"Rongxin Mi","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087772","display_name":"National Computer Network Emergency Response Technical Team/Coordination Center of Chinar","ror":"https://ror.org/00247dh76","country_code":"CN","type":"nonprofit","lineage":["https://openalex.org/I4210087772"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rongxin Mi","raw_affiliation_strings":["National Computer Network Emergency Response Technical Team/Coordination Center of China,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Computer Network Emergency Response Technical Team/Coordination Center of China,Beijing,China","institution_ids":["https://openalex.org/I4210087772"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064741421","display_name":"Nannan Sun","orcid":"https://orcid.org/0000-0002-5255-9784"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nannan Sun","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102783832","display_name":"Dakui Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dakui Wang","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101651401","display_name":"Yanbing Liu","orcid":"https://orcid.org/0000-0001-8068-8316"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanbing Liu","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences,Beijing,China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1036","last_page":"1041"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9842000007629395,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9842000007629395,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9767000079154968,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9563999772071838,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.7889168858528137},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6758767366409302},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.6406810879707336},{"id":"https://openalex.org/keywords/shot","display_name":"Shot (pellet)","score":0.581553041934967},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5374916195869446},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4400455355644226},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.32709774374961853},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.21267691254615784},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.07493254542350769}],"concepts":[{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.7889168858528137},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6758767366409302},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6406810879707336},{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.581553041934967},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5374916195869446},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4400455355644226},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32709774374961853},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21267691254615784},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.07493254542350769},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/smc54092.2024.10831406","is_oa":false,"landing_page_url":"https://doi.org/10.1109/smc54092.2024.10831406","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Systems, Man, and Cybernetics (SMC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3084162778","display_name":null,"funder_award_id":"2023YFC3303800","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W2156150815","https://openalex.org/W3168125510","https://openalex.org/W4382406466","https://openalex.org/W4388999041","https://openalex.org/W4392909055","https://openalex.org/W4400076706","https://openalex.org/W6677103964","https://openalex.org/W6727099177","https://openalex.org/W6769627184","https://openalex.org/W6774302960","https://openalex.org/W6779124799","https://openalex.org/W6780482815","https://openalex.org/W6780665282","https://openalex.org/W6790943123","https://openalex.org/W6794457956","https://openalex.org/W6796581206","https://openalex.org/W6810362000","https://openalex.org/W6811340617","https://openalex.org/W6811466168","https://openalex.org/W6842258392","https://openalex.org/W6842758154","https://openalex.org/W6842795023","https://openalex.org/W6846163848","https://openalex.org/W6848451824","https://openalex.org/W6850240752","https://openalex.org/W6850625674","https://openalex.org/W6851762504","https://openalex.org/W6852927819","https://openalex.org/W6853048723","https://openalex.org/W6853369108","https://openalex.org/W6854094408","https://openalex.org/W6855780230","https://openalex.org/W6857551316","https://openalex.org/W6862021088","https://openalex.org/W6893640197"],"related_works":["https://openalex.org/W2074502265","https://openalex.org/W4214877189","https://openalex.org/W2773965352","https://openalex.org/W2381179799","https://openalex.org/W2980279061","https://openalex.org/W2334685461","https://openalex.org/W2366718574","https://openalex.org/W2359774528","https://openalex.org/W4298312966","https://openalex.org/W2325697621"],"abstract_inverted_index":{"Model":[0],"pruning,":[1],"as":[2,56],"an":[3],"effective":[4],"method":[5,77],"for":[6,36,108],"compressing":[7],"large":[8,84],"language":[9,22,85,185],"models":[10,86],"(LLMs),":[11],"has":[12],"recently":[13],"attracted":[14],"considerable":[15],"attention":[16],"in":[17,95],"the":[18,109,121,171,192],"field":[19],"of":[20,83,113,123,130,142,178],"natural":[21],"processing.":[23],"However,":[24],"existing":[25],"LLM":[26],"pruning":[27,35,49,76,82,102,129,155],"methods":[28],"have":[29],"two":[30],"main":[31],"drawbacks:":[32],"(1)":[33],"Iterative":[34],"LLMs":[37,52],"with":[38,87],"over":[39],"a":[40,57,74,100,175],"billion":[41],"parameters":[42],"requires":[43],"retraining,":[44],"which":[45,90],"leads":[46],"to":[47,126,145,157,169],"significant":[48],"costs.":[50],"(2)":[51],"Pruning":[53],"is":[54],"formalized":[55],"weight":[58,124,137],"reconstruction":[59],"problem":[60],"that":[61,189],"necessitates":[62],"second-order":[63],"information,":[64],"incurring":[65],"expensive":[66],"computations.":[67],"To":[68],"address":[69],"these":[70],"issues,":[71],"we":[72,98,134,150,163],"propose":[73],"novel":[75,101],"named":[78],"Eplra:":[79],"efficient":[80],"one-shot":[81,111],"low-rank":[88,165],"approximation,":[89],"efficiently":[91],"identifies":[92],"sparse":[93],"networks":[94],"LLMs.":[96,114],"Specifically,":[97],"design":[99],"metric":[103],"based":[104],"on":[105,183],"input":[106,118],"activations":[107,119],"rapid":[110],"compression":[112],"We":[115],"first":[116],"incorporate":[117],"into":[120,153],"calculation":[122],"importance":[125],"promote":[127],"precise":[128],"low-priority":[131],"weights.":[132],"Then,":[133],"perform":[135],"local":[136],"comparisons":[138],"across":[139],"each":[140],"output":[141],"linear":[143],"layers":[144],"induce":[146],"uniform":[147],"sparsity.":[148],"Next,":[149],"expand":[151],"Eplra":[152,190],"semi-structured":[154],"patterns":[156],"accommodate":[158],"various":[159,184],"acceleration":[160],"scenarios.":[161],"Finally,":[162],"employ":[164],"parametrized":[166],"update":[167],"matrices":[168],"fine-tune":[170],"pruned":[172],"model,":[173],"facilitating":[174],"swift":[176],"recovery":[177],"model":[179],"performance.":[180],"Experimental":[181],"results":[182],"benchmark":[186],"datasets":[187],"demonstrate":[188],"outperforms":[191],"state-of-the-art":[193],"methods.":[194]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}