{"id":"https://openalex.org/W4391594351","doi":"https://doi.org/10.1109/lca.2024.3363492","title":"eDKM: An Efficient and Accurate Train-Time Weight Clustering for Large Language Models","display_name":"eDKM: An Efficient and Accurate Train-Time Weight Clustering for Large Language Models","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4391594351","doi":"https://doi.org/10.1109/lca.2024.3363492"},"language":"en","primary_location":{"id":"doi:10.1109/lca.2024.3363492","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lca.2024.3363492","pdf_url":null,"source":{"id":"https://openalex.org/S17643076","display_name":"IEEE Computer Architecture Letters","issn_l":"1556-6056","issn":["1556-6056","1556-6064","2473-2575"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Computer Architecture Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115076654","display_name":"Minsik Cho","orcid":"https://orcid.org/0000-0003-0481-2682"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Minsik Cho","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-0481-2682","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023280155","display_name":"Keivan Alizadeh Vahid","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Keivan A. Vahid","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0009-0001-2703-6674","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101753713","display_name":"Qichen Fu","orcid":"https://orcid.org/0009-0008-5976-4095"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qichen Fu","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0009-0008-5976-4095","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068392026","display_name":"Saurabh Adya","orcid":"https://orcid.org/0009-0000-4533-6577"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saurabh Adya","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0009-0000-4533-6577","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073050611","display_name":"Carlo C. Del Mundo","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carlo C. Del Mundo","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0009-0009-0991-1808","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056246621","display_name":"Mohammad Rastegari","orcid":"https://orcid.org/0000-0001-9606-3687"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammad Rastegari","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0000-0001-9606-3687","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113880378","display_name":"Devang Naik","orcid":"https://orcid.org/0009-0007-7838-1623"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devang Naik","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0009-0007-7838-1623","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031481674","display_name":"Peter Zatloukal","orcid":"https://orcid.org/0009-0000-0451-0591"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Zatloukal","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0009-0000-0451-0591","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210153776"],"apc_list":null,"apc_paid":null,"fwci":1.3892,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.82768761,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"23","issue":"1","first_page":"37","last_page":"40"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7894577980041504},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.7478118538856506},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.685096263885498},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.6156352162361145},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5551738142967224},{"id":"https://openalex.org/keywords/compression-ratio","display_name":"Compression ratio","score":0.5530991554260254},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.49779582023620605},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.46983620524406433},{"id":"https://openalex.org/keywords/compression","display_name":"Compression (physics)","score":0.4591199457645416},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.4568122625350952},{"id":"https://openalex.org/keywords/computer-hardware","display_name":"Computer hardware","score":0.3305826485157013},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3304826021194458},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.27574047446250916},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11544844508171082},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.09308364987373352}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7894577980041504},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.7478118538856506},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.685096263885498},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.6156352162361145},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5551738142967224},{"id":"https://openalex.org/C25797200","wikidata":"https://www.wikidata.org/wiki/Q828137","display_name":"Compression ratio","level":3,"score":0.5530991554260254},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.49779582023620605},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.46983620524406433},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.4591199457645416},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.4568122625350952},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.3305826485157013},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3304826021194458},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27574047446250916},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11544844508171082},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.09308364987373352},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C511840579","wikidata":"https://www.wikidata.org/wiki/Q12757","display_name":"Internal combustion engine","level":2,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C171146098","wikidata":"https://www.wikidata.org/wiki/Q124192","display_name":"Automotive engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lca.2024.3363492","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lca.2024.3363492","pdf_url":null,"source":{"id":"https://openalex.org/S17643076","display_name":"IEEE Computer Architecture Letters","issn_l":"1556-6056","issn":["1556-6056","1556-6064","2473-2575"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Computer Architecture Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W2300242332","https://openalex.org/W2962761403","https://openalex.org/W2963145730","https://openalex.org/W2982479999","https://openalex.org/W6684859321","https://openalex.org/W6733877748","https://openalex.org/W6748163181","https://openalex.org/W6751769266","https://openalex.org/W6765643347","https://openalex.org/W6770425567","https://openalex.org/W6771140128","https://openalex.org/W6771369672","https://openalex.org/W6775717222","https://openalex.org/W6800888104","https://openalex.org/W6810738896","https://openalex.org/W6811340617","https://openalex.org/W6846164622","https://openalex.org/W6850625674","https://openalex.org/W6853048723","https://openalex.org/W6853176743"],"related_works":["https://openalex.org/W2928062709","https://openalex.org/W4383723869","https://openalex.org/W2161302774","https://openalex.org/W2388481516","https://openalex.org/W4384298135","https://openalex.org/W4383722264","https://openalex.org/W1723410974","https://openalex.org/W3007688875","https://openalex.org/W2766695209","https://openalex.org/W3123970444"],"abstract_inverted_index":{"The":[0],"size":[1],"of":[2,6,24,29,102,106,121,176],"LLMs":[3],"(i.e.,":[4,190],"billions":[5],"parameters)":[7],"requires":[8],"highly":[9],"effective":[10],"compression":[11,19,63],"to":[12,77,79,97,112,142,162],"fit":[13],"into":[14],"storage-limited":[15],"devices.":[16],"Among":[17],"many":[18],"techniques,":[20],"weight-clustering,":[21],"a":[22,88,109,154,177],"form":[23],"non-linear":[25],"quantization,":[26],"is":[27,45,136],"one":[28],"the":[30,59,99,118,125,167,172],"leading":[31],"candidates":[32],"for":[33,48,117,192,195],"LLM":[34,49,81,188],"compression,":[35],"and":[36,65,130,152,197],"supported":[37],"by":[38,94,104,127,170,180],"modern":[39],"smartphones.":[40],"Yet,":[41],"its":[42,69],"training":[43],"overhead":[44],"prohibitively":[46],"significant":[47],"fine-tuning.":[50],"Especially,":[51],"Differentiable":[52],"KMeans":[53],"Clustering,":[54],"or":[55],"DKM,":[56,122],"has":[57],"shown":[58],"state-of-the-art":[60],"trade-off":[61],"between":[62],"ratio":[64],"accuracy":[66,185],"regression,":[67],"but":[68],"large":[70],"memory":[71,100,174],"complexity":[72],"makes":[73],"it":[74],"nearly":[75],"impossible":[76],"apply":[78],"train-time":[80,173],"compression.":[82],"In":[83],"this":[84],"paper,":[85],"we":[86,123],"propose":[87],"memory-efficient":[89],"DKM":[90,103],"implementation,":[91],"eDKM":[92,149],"powered":[93],"novel":[95],"techniques":[96],"reduce":[98],"footprint":[101,175],"orders":[105],"magnitudes.":[107],"For":[108],"given":[110],"tensor":[111,126,139],"be":[113],"saved":[114],"on":[115,186],"CPU":[116],"backward":[119],"pass":[120],"compressed":[124],"applying":[128],"uniquification":[129],"sharding":[131],"after":[132],"checking":[133],"if":[134],"there":[135],"no":[137],"duplicated":[138],"previously":[140],"copied":[141],"CPU.":[143],"Our":[144],"experimental":[145],"results":[146],"demonstrate":[147],"that":[148],"can":[150],"fine-tune":[151],"compress":[153],"pretrained":[155],"LLaMA":[156],"7B":[157],"model":[158],"from":[159],"12.6":[160],"GB":[161,164],"2.5":[163],"(3bit/weight)":[165],"with":[166],"Alpaca":[168],"dataset":[169],"reducing":[171],"decoder":[178],"layer":[179],"130\u00d7,":[181],"while":[182],"delivering":[183],"good":[184],"broader":[187],"benchmarks":[189],"77.7%":[191],"PIQA,":[193],"66.1%":[194],"Winograde,":[196],"so":[198],"on).":[199]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}