{"id":"https://openalex.org/W7129089911","doi":"https://doi.org/10.1109/tpami.2026.3664873","title":"MC#: Mixture Compressor for Mixture-of-Experts Large Models","display_name":"MC#: Mixture Compressor for Mixture-of-Experts Large Models","publication_year":2026,"publication_date":"2026-02-16","ids":{"openalex":"https://openalex.org/W7129089911","doi":"https://doi.org/10.1109/tpami.2026.3664873","pmid":"https://pubmed.ncbi.nlm.nih.gov/41697807"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2026.3664873","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3664873","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5126091291","display_name":"Wei Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210086892","display_name":"Education University of Hong Kong","ror":"https://ror.org/000t0f062","country_code":"HK","type":"education","lineage":["https://openalex.org/I4210086892"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wei Huang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China"],"raw_orcid":"https://orcid.org/0009-0007-9885-0028","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I4210086892","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002061412","display_name":"Yue Liao","orcid":"https://orcid.org/0000-0002-2671-0655"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yue Liao","raw_affiliation_strings":["School of Computing, National University of Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-2671-0655","affiliations":[{"raw_affiliation_string":"School of Computing, National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126151107","display_name":"Yukang Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yukang Chen","raw_affiliation_strings":["NVIDIA Research, Santa Clara, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA Research, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126114884","display_name":"Jianhui Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210086892","display_name":"Education University of Hong Kong","ror":"https://ror.org/000t0f062","country_code":"HK","type":"education","lineage":["https://openalex.org/I4210086892"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jianhui Liu","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I4210086892","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054734725","display_name":"Haoru Tan","orcid":"https://orcid.org/0009-0001-6721-2468"},"institutions":[{"id":"https://openalex.org/I4210086892","display_name":"Education University of Hong Kong","ror":"https://ror.org/000t0f062","country_code":"HK","type":"education","lineage":["https://openalex.org/I4210086892"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Haoru Tan","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China"],"raw_orcid":"https://orcid.org/0009-0001-6721-2468","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I4210086892","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126077462","display_name":"Si Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Si Liu","raw_affiliation_strings":["Institute of Artificial Intelligence, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9180-2935","affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126096816","display_name":"Shiming Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210086892","display_name":"Education University of Hong Kong","ror":"https://ror.org/000t0f062","country_code":"HK","type":"education","lineage":["https://openalex.org/I4210086892"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shiming Zhang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I4210086892","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shuicheng Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Shuicheng Yan","raw_affiliation_strings":["School of Computing, National University of Singapore, Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing, National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"last","author":{"id":null,"display_name":"Xiaojuan Qi","orcid":"https://orcid.org/0000-0002-4285-1626"},"institutions":[{"id":"https://openalex.org/I4210086892","display_name":"Education University of Hong Kong","ror":"https://ror.org/000t0f062","country_code":"HK","type":"education","lineage":["https://openalex.org/I4210086892"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiaojuan Qi","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China"],"raw_orcid":"https://orcid.org/0000-0002-4285-1626","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I4210086892","https://openalex.org/I889458895"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.19480728,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"48","issue":"7","first_page":"7607","last_page":"7621"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.30329999327659607,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.30329999327659607,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.28290000557899475,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.20170000195503235,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.603600025177002},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5020999908447266},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.44209998846054077},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.38429999351501465},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3612000048160553},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.35690000653266907},{"id":"https://openalex.org/keywords/minification","display_name":"Minification","score":0.34389999508857727},{"id":"https://openalex.org/keywords/expert-system","display_name":"Expert system","score":0.33970001339912415}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7807999849319458},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.603600025177002},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5020999908447266},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.44209998846054077},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4113999903202057},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.38429999351501465},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3743000030517578},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3612000048160553},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.35690000653266907},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.34389999508857727},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.33970001339912415},{"id":"https://openalex.org/C99138194","wikidata":"https://www.wikidata.org/wiki/Q183427","display_name":"Hash function","level":2,"score":0.33660000562667847},{"id":"https://openalex.org/C46900642","wikidata":"https://www.wikidata.org/wiki/Q2647","display_name":"Huffman coding","level":3,"score":0.29829999804496765},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.272599995136261},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.2678000032901764},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.26589998602867126},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.2632000148296356},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C155846161","wikidata":"https://www.wikidata.org/wiki/Q1143367","display_name":"Graphical model","level":2,"score":0.2549999952316284},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2026.3664873","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3664873","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:41697807","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41697807","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2300242332","https://openalex.org/W2963363373","https://openalex.org/W4385571917","https://openalex.org/W4391136507","https://openalex.org/W4402670692","https://openalex.org/W4402671637","https://openalex.org/W4402671766","https://openalex.org/W4403791888","https://openalex.org/W4404782328","https://openalex.org/W4404918643","https://openalex.org/W4412945475","https://openalex.org/W4413146209","https://openalex.org/W4414281281","https://openalex.org/W4415799245","https://openalex.org/W7124471359","https://openalex.org/W7160708377"],"related_works":[],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"has":[2],"emerged":[3],"as":[4,133,174],"an":[5,226],"effective":[6],"and":[7,16,52,61,74,97,108,119,145,155],"efficient":[8,263],"scaling":[9],"mechanism":[10],"for":[11,81,190,262],"large":[12],"language":[13],"models":[14,18,169],"(LLMs)":[15],"vision-language":[17],"(VLMs).":[19],"By":[20,198],"expanding":[21],"a":[22,78,90,134,149,175,186,220,233],"single":[23],"feed-forward":[24],"network":[25],"into":[26,50],"multiple":[27,54],"expert":[28,65,99,143,170,249],"branches,":[29],"MoE":[30],"increases":[31],"model":[32,72,153],"capacity":[33],"while":[34],"maintaining":[35],"efficiency":[36],"through":[37],"sparse":[38],"activation.":[39,197],"However,":[40],"despite":[41],"this":[42],"sparsity,":[43],"the":[44,68,103,242],"need":[45],"to":[46,71,110,241],"preload":[47],"all":[48],"experts":[49,55,107,189],"memory":[51,62],"activate":[53],"per":[56,172],"input":[57],"introduces":[58],"significant":[59],"computational":[60],"overhead.":[63],"The":[64,138],"module":[66],"becomes":[67],"dominant":[69],"contributor":[70],"size":[73,154],"inference":[75],"cost,":[76],"posing":[77],"major":[79],"challenge":[80],"deployment.":[82],"To":[83,116,157],"address":[84],"this,":[85],"we":[86,122,161],"propose":[87],"MC#":[88,208,218],"(Mixture-Compressor-sharp),":[89],"unified":[91],"framework":[92],"that":[93],"combines":[94],"static":[95,201],"quantization":[96,146],"dynamic":[98,206],"pruning":[100],"by":[101,251],"leveraging":[102],"significance":[104],"of":[105,114,188,228,265],"both":[106],"tokens":[109],"achieve":[111],"aggressive":[112],"compression":[113,211],"MoE-LLMs/VLMs.":[115],"reduce":[117,158],"storage":[118],"loading":[120],"overhead,":[121],"introduce":[123,163],"Pre-Loading":[124],"Mixed-Precision":[125],"Quantization":[126],"(PMQ),":[127],"which":[128,168],"formulates":[129],"adaptive":[130],"bit":[131],"allocation":[132],"linear":[135],"programming":[136],"problem.":[137],"objective":[139],"function":[140],"jointly":[141],"considers":[142],"importance":[144],"error,":[147],"producing":[148],"Pareto-optimal":[150],"trade-off":[151],"between":[152],"performance.":[156],"runtime":[159],"computation,":[160],"further":[162,247],"Online":[164],"Top-any":[165],"Pruning":[166],"(OTP),":[167],"activation":[171,250],"token":[173],"learnable":[176],"distribution":[177],"via":[178],"Gumbel-Softmax":[179],"sampling.":[180],"During":[181],"inference,":[182],"OTP":[183,246],"dynamically":[184],"selects":[185],"subset":[187],"each":[191],"token,":[192],"allowing":[193],"fine-grained":[194],"control":[195],"over":[196],"combining":[199],"PMQ's":[200],"bit-width":[202],"optimization":[203],"with":[204,212,231,253],"OTP's":[205],"routing,":[207],"achieves":[209,219],"extreme":[210],"minimal":[213],"accuracy":[214],"degradation.":[215],"On":[216],"DeepSeek-VL2,":[217],"6.2":[221],"\u00d7":[222],"weight":[223],"reduction":[224],"at":[225],"average":[227],"2.57":[229],"bits,":[230],"only":[232],"1.7%":[234],"drop":[235],"across":[236],"five":[237],"multimodal":[238],"benchmarks":[239],"compared":[240],"16-bit":[243],"baseline.":[244],"Moreover,":[245],"reduces":[248],"20%":[252],"less":[254],"than":[255],"1%":[256],"performance":[257],"loss,":[258],"demonstrating":[259],"strong":[260],"potential":[261],"deployment":[264],"MoE-based":[266],"models.":[267]},"counts_by_year":[],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2026-02-17T00:00:00"}