{"id":"https://openalex.org/W4416203772","doi":"https://doi.org/10.1145/3712285.3759903","title":"Diff-MoE: Efficient Batched MoE Inference with Priority-Driven Differential Expert Caching","display_name":"Diff-MoE: Efficient Batched MoE Inference with Priority-Driven Differential Expert Caching","publication_year":2025,"publication_date":"2025-11-12","ids":{"openalex":"https://openalex.org/W4416203772","doi":"https://doi.org/10.1145/3712285.3759903"},"language":null,"primary_location":{"id":"doi:10.1145/3712285.3759903","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3712285.3759903","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100329685","display_name":"Kexin Li","orcid":"https://orcid.org/0009-0007-8431-0864"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kexin Li","raw_affiliation_strings":["National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0007-8431-0864","affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108120207","display_name":"Wenkan Huang","orcid":"https://orcid.org/0009-0005-0581-522X"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenkan Huang","raw_affiliation_strings":["National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0005-0581-522X","affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115599465","display_name":"Qinggang Wang","orcid":"https://orcid.org/0000-0002-9951-3345"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qinggang Wang","raw_affiliation_strings":["National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-9951-3345","affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023600493","display_name":"Long Zheng","orcid":"https://orcid.org/0000-0001-7903-2061"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Long Zheng","raw_affiliation_strings":["National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0001-7903-2061","affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022398389","display_name":"Xiaofei Liao","orcid":"https://orcid.org/0000-0001-6302-813X"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaofei Liao","raw_affiliation_strings":["National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0001-6302-813X","affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022262922","display_name":"Hai Jin","orcid":"https://orcid.org/0000-0002-3934-7605"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hai Jin","raw_affiliation_strings":["National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-3934-7605","affiliations":[{"raw_affiliation_string":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024664385","display_name":"Jingling Xue","orcid":"https://orcid.org/0000-0003-0380-3506"},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Jingling Xue","raw_affiliation_strings":["School of Computer Science and Engineering, University of New South Wales, Sydney, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0003-0380-3506","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, University of New South Wales, Sydney, NSW, Australia","institution_ids":["https://openalex.org/I31746571"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.8873,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.88825215,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1951","last_page":"1965"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.21320000290870667,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.21320000290870667,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.17800000309944153,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.10589999705553055,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.7300999760627747},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7092999815940857},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.625},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.6194000244140625},{"id":"https://openalex.org/keywords/locality","display_name":"Locality","score":0.5184999704360962},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.49790000915527344},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.44440001249313354},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.44179999828338623},{"id":"https://openalex.org/keywords/cpu-cache","display_name":"CPU cache","score":0.4207000136375427}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8292999863624573},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.7300999760627747},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7092999815940857},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.625},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.6194000244140625},{"id":"https://openalex.org/C2779808786","wikidata":"https://www.wikidata.org/wiki/Q6664603","display_name":"Locality","level":2,"score":0.5184999704360962},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.49790000915527344},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.44440001249313354},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.44179999828338623},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.4207000136375427},{"id":"https://openalex.org/C133588205","wikidata":"https://www.wikidata.org/wiki/Q28455645","display_name":"Instruction prefetch","level":3,"score":0.4027999937534332},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.3874000012874603},{"id":"https://openalex.org/C2778100165","wikidata":"https://www.wikidata.org/wiki/Q1589327","display_name":"Memory hierarchy","level":3,"score":0.37689998745918274},{"id":"https://openalex.org/C5165142","wikidata":"https://www.wikidata.org/wiki/Q5432732","display_name":"False sharing","level":5,"score":0.3686999976634979},{"id":"https://openalex.org/C126831891","wikidata":"https://www.wikidata.org/wiki/Q221673","display_name":"Host (biology)","level":2,"score":0.36660000681877136},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.3594000041484833},{"id":"https://openalex.org/C38556500","wikidata":"https://www.wikidata.org/wiki/Q13404475","display_name":"Cache algorithms","level":4,"score":0.3292999863624573},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3109000027179718},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.30880001187324524},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3068000078201294},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.27970001101493835},{"id":"https://openalex.org/C27602214","wikidata":"https://www.wikidata.org/wiki/Q1868547","display_name":"Locality of reference","level":3,"score":0.27410000562667847},{"id":"https://openalex.org/C93226319","wikidata":"https://www.wikidata.org/wiki/Q193137","display_name":"Differential (mechanical device)","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2596000134944916},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.25679999589920044}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3712285.3759903","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3712285.3759903","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G701482840","display_name":null,"funder_award_id":"62402456,62402457","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7794448185","display_name":null,"funder_award_id":"2023YFB4503400","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W2888482885","https://openalex.org/W2963341956","https://openalex.org/W2963748441","https://openalex.org/W2964223283","https://openalex.org/W3034999214","https://openalex.org/W3042639142","https://openalex.org/W3129307903","https://openalex.org/W3130689885","https://openalex.org/W4214493665","https://openalex.org/W4280525524","https://openalex.org/W4307079201","https://openalex.org/W4308083754","https://openalex.org/W4321636575","https://openalex.org/W4327810158","https://openalex.org/W4366090965","https://openalex.org/W4376226279","https://openalex.org/W4387389714","https://openalex.org/W4390438324","https://openalex.org/W4390723197","https://openalex.org/W4392221718","https://openalex.org/W4396817294","https://openalex.org/W4401211627","https://openalex.org/W4402671637","https://openalex.org/W4402671950","https://openalex.org/W4404133527","https://openalex.org/W4404134154","https://openalex.org/W4404341652","https://openalex.org/W4404792840","https://openalex.org/W4404955001","https://openalex.org/W4404955792","https://openalex.org/W4407423377","https://openalex.org/W4414849752"],"related_works":[],"abstract_inverted_index":{"The":[0],"emerging":[1],"Mixture-of-Experts":[2],"(MoE)":[3],"model":[4],"mitigates":[5],"the":[6,142,165],"high":[7],"compute":[8],"cost":[9],"of":[10,18,77,85],"large-scale":[11],"LLMs":[12],"by":[13,185],"sparsely":[14],"activating":[15],"a":[16,30,82,111,137,156],"subset":[17],"experts":[19,47,86,120,145,161],"during":[20],"inference.":[21],"However,":[22],"MoE":[23,167],"requires":[24],"storing":[25],"massive":[26],"expert":[27,80],"parameters,":[28],"creating":[29],"severe":[31],"memory":[32,43,50],"bottleneck":[33],"on":[34,151],"resource-constrained":[35],"GPUs.":[36],"Existing":[37],"approaches":[38],"offload":[39],"parameters":[40],"to":[41,48,59,173],"host":[42],"and":[44,62,141,149,190,197],"prefetch":[45],"activated":[46],"GPU":[49,116],"with":[51,171],"sophisticated":[52],"policies,":[53],"but":[54],"these":[55],"solutions":[56],"are":[57,87,129,146],"tailored":[58],"single-batch":[60],"inference":[61,91,183],"suffer":[63],"from":[64],"communication":[65],"bottlenecks":[66],"at":[67],"larger":[68],"batch":[69],"sizes,":[70],"limiting":[71],"throughput.":[72],"We":[73],"identify":[74],"two":[75],"forms":[76],"locality":[78],"in":[79,115,122,132,164],"activation:":[81],"small":[83],"set":[84],"frequently":[88],"invoked":[89],"across":[90],"(global":[92],"locality),":[93],"while":[94],"others":[95],"recur":[96],"within":[97],"short":[98],"decoding":[99],"bursts":[100],"(temporal":[101],"locality).":[102],"To":[103],"exploit":[104],"this,":[105],"we":[106],"propose":[107],"Diff-MoE,":[108],"which":[109],"introduces":[110],"differential":[112],"cache":[113],"hierarchy":[114],"memory.":[117],"Globally":[118],"hot":[119,127],"reside":[121],"per-layer":[123,133],"high-priority":[124],"caches,":[125],"locally":[126],"ones":[128],"dynamically":[130],"managed":[131],"medium-priority":[134],"caches":[135],"under":[136],"priority-driven":[138],"replacement":[139],"policy,":[140],"remaining":[143],"cold":[144],"cached":[147],"temporarily":[148],"evicted":[150],"demand.":[152],"Moreover,":[153],"Diff-MoE":[154,181],"incorporates":[155],"lightweight":[157],"predictor":[158],"that":[159,180],"prefetches":[160],"likely":[162],"needed":[163],"next":[166],"layer,":[168],"overlapping":[169],"migration":[170],"computation":[172],"further":[174],"reduce":[175],"latency.":[176],"Our":[177],"evaluation":[178],"shows":[179],"improves":[182],"throughput":[184],"2.74":[186],"\u00d7,":[187,189],"2.22":[188],"1.55":[191],"\u00d7":[192],"over":[193],"DeepSpeed,":[194],"Pre-gated":[195],"MoE,":[196],"MoE-Infinity,":[198],"respectively.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-11-12T00:00:00"}