{"id":"https://openalex.org/W4409158695","doi":"https://doi.org/10.1145/3690624.3709196","title":"ResMoE: Space-efficient Compression of Mixture of Experts LLMs via Residual Restoration","display_name":"ResMoE: Space-efficient Compression of Mixture of Experts LLMs via Residual Restoration","publication_year":2025,"publication_date":"2025-04-04","ids":{"openalex":"https://openalex.org/W4409158695","doi":"https://doi.org/10.1145/3690624.3709196"},"language":"en","primary_location":{"id":"doi:10.1145/3690624.3709196","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3690624.3709196","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3690624.3709196","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116926198","display_name":"Mengting Ai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mengting Ai","raw_affiliation_strings":["UIUC, Champaign, IL, USA"],"raw_orcid":"https://orcid.org/0009-0009-3878-6188","affiliations":[{"raw_affiliation_string":"UIUC, Champaign, IL, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024869027","display_name":"Tianxin Wei","orcid":"https://orcid.org/0000-0003-4450-2005"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tianxin Wei","raw_affiliation_strings":["UIUC, Champaign, IL, USA"],"raw_orcid":"https://orcid.org/0000-0003-4450-2005","affiliations":[{"raw_affiliation_string":"UIUC, Champaign, IL, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043775295","display_name":"Yifan Chen","orcid":"https://orcid.org/0000-0001-9462-9122"},"institutions":[{"id":"https://openalex.org/I141568987","display_name":"Hong Kong Baptist University","ror":"https://ror.org/0145fw131","country_code":"HK","type":"education","lineage":["https://openalex.org/I141568987"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yifan Chen","raw_affiliation_strings":["HKBU, Kowloon, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0001-9462-9122","affiliations":[{"raw_affiliation_string":"HKBU, Kowloon, Hong Kong","institution_ids":["https://openalex.org/I141568987"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055176391","display_name":"Zhichen Zeng","orcid":"https://orcid.org/0000-0002-5534-3401"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhichen Zeng","raw_affiliation_strings":["UIUC, Champaign, IL, USA"],"raw_orcid":"https://orcid.org/0000-0002-5534-3401","affiliations":[{"raw_affiliation_string":"UIUC, Champaign, IL, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033038201","display_name":"Ritchie Zhao","orcid":"https://orcid.org/0000-0003-1656-9165"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ritchie Zhao","raw_affiliation_strings":["NVIDIA, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-1656-9165","affiliations":[{"raw_affiliation_string":"NVIDIA, Seattle, WA, USA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013420358","display_name":"Girish V. Varatkar","orcid":"https://orcid.org/0009-0000-4504-0717"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Girish Varatkar","raw_affiliation_strings":["Apple, Cupertino, CA, USA"],"raw_orcid":"https://orcid.org/0009-0000-4504-0717","affiliations":[{"raw_affiliation_string":"Apple, Cupertino, CA, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035073741","display_name":"Bita Darvish Rouhani","orcid":"https://orcid.org/0000-0002-8412-4320"},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bita Darvish Rouhani","raw_affiliation_strings":["NVIDIA, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8412-4320","affiliations":[{"raw_affiliation_string":"NVIDIA, Seattle, WA, USA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102008510","display_name":"Xianfeng Tang","orcid":"https://orcid.org/0000-0003-1554-2761"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xianfeng Tang","raw_affiliation_strings":["Amazon, Palo Alto, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-1554-2761","affiliations":[{"raw_affiliation_string":"Amazon, Palo Alto, CA, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068043486","display_name":"Hanghang Tong","orcid":"https://orcid.org/0000-0003-4405-3887"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hanghang Tong","raw_affiliation_strings":["UIUC, Champaign, IL, USA"],"raw_orcid":"https://orcid.org/0000-0003-4405-3887","affiliations":[{"raw_affiliation_string":"UIUC, Champaign, IL, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073158087","display_name":"Jingrui He","orcid":"https://orcid.org/0000-0002-6429-6272"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jingrui He","raw_affiliation_strings":["UIUC, Champaign, IL, USA"],"raw_orcid":"https://orcid.org/0000-0002-6429-6272","affiliations":[{"raw_affiliation_string":"UIUC, Champaign, IL, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.402,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.98455606,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"12"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9731000065803528,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11363","display_name":"Dental Radiography and Imaging","score":0.9397000074386597,"subfield":{"id":"https://openalex.org/subfields/3504","display_name":"Oral Surgery"},"field":{"id":"https://openalex.org/fields/35","display_name":"Dentistry"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.6312711238861084},{"id":"https://openalex.org/keywords/compression","display_name":"Compression (physics)","score":0.550387442111969},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.5032843947410583},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.4157279133796692},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.323822021484375},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.2352847456932068},{"id":"https://openalex.org/keywords/composite-material","display_name":"Composite material","score":0.21143969893455505},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.111379474401474}],"concepts":[{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.6312711238861084},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.550387442111969},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.5032843947410583},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.4157279133796692},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.323822021484375},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.2352847456932068},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.21143969893455505},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.111379474401474},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3690624.3709196","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3690624.3709196","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.06881","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.06881","pdf_url":"https://arxiv.org/pdf/2503.06881","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3690624.3709196","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3690624.3709196","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W2251939518","https://openalex.org/W2295512376","https://openalex.org/W2606555609","https://openalex.org/W2963015836","https://openalex.org/W2963846996","https://openalex.org/W2978670439","https://openalex.org/W2990138404","https://openalex.org/W2998617917","https://openalex.org/W3034368386","https://openalex.org/W3093502611","https://openalex.org/W3131500599","https://openalex.org/W3138154797","https://openalex.org/W3138516171","https://openalex.org/W3194676777","https://openalex.org/W4214614183","https://openalex.org/W4221146568","https://openalex.org/W4226087293","https://openalex.org/W4238473618","https://openalex.org/W4281560842","https://openalex.org/W4281613206","https://openalex.org/W4283314431","https://openalex.org/W4367046806","https://openalex.org/W4376983316","https://openalex.org/W4382322807","https://openalex.org/W4385565718","https://openalex.org/W4387846361","https://openalex.org/W4389524291","https://openalex.org/W4390671319","https://openalex.org/W4393160571","https://openalex.org/W4396723267","https://openalex.org/W4396758529","https://openalex.org/W4402671950","https://openalex.org/W6600195515","https://openalex.org/W6603297324","https://openalex.org/W6606988362"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2560215812","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2949601986"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"Transformer,":[2,125],"the":[3,43,69,76,81,105,116,137,159],"backbone":[4],"architecture":[5],"of":[6,18,86,118,139],"multiple":[7],"phenomenal":[8],"language":[9,112],"models,":[10],"leverages":[11],"sparsity":[12],"by":[13,144],"activating":[14],"only":[15],"a":[16,62,91],"fraction":[17],"model":[19,44],"parameters":[20,45,140],"for":[21,84,107],"each":[22],"input":[23],"token.":[24],"The":[25,130,152],"sparse":[26],"structure,":[27],"while":[28,98,148],"allowing":[29],"constant":[30],"time":[31],"costs,":[32],"results":[33,131],"in":[34,90,141],"space":[35,82],"inefficiency:":[36],"we":[37],"still":[38],"need":[39],"to":[40,60,110,146],"load":[41],"all":[42],"during":[46],"inference.":[47],"We":[48,114],"introduce":[49],"ResMoE,":[50],"an":[51,142],"innovative":[52],"MoE":[53,88],"approximation":[54],"framework":[55],"that":[56,133],"utilizes":[57],"Wasserstein":[58],"barycenter":[59,73],"extract":[61],"common":[63],"expert":[64,74,143],"(barycenter":[65],"expert)":[66],"and":[67,75,93,127,158],"approximate":[68],"residuals":[70],"between":[71],"this":[72],"original":[77],"ones.":[78],"ResMoE":[79,119,134],"enhances":[80],"efficiency":[83],"inference":[85],"large-scale":[87],"Transformers":[89],"one-shot":[92],"data-agnostic":[94],"manner":[95],"without":[96],"retraining":[97],"maintaining":[99,149],"minimal":[100],"accuracy":[101],"loss,":[102],"thereby":[103],"paving":[104],"way":[106],"broader":[108],"accessibility":[109],"large":[111],"models.":[113,129],"demonstrate":[115],"effectiveness":[117],"through":[120],"extensive":[121],"experiments":[122],"on":[123],"Switch":[124],"Mixtral,":[126],"DeepSeekMoE":[128],"show":[132],"can":[135],"reduce":[136],"number":[138],"up":[145],"75%":[147],"comparable":[150],"performance.":[151],"code":[153],"is":[154,162],"available":[155,163],"at":[156,164],"https://github.com/iDEA-iSAIL-Lab-UIUC/ResMoE,":[157],"supplementary":[160],"appendix":[161],"https://famous-blue-raincoat.github.io/mengtingai/files/ResMoE_Appendix.pdf.":[165]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":2}],"updated_date":"2026-08-08T01:25:22.217667","created_date":"2025-10-10T00:00:00"}