{"id":"https://openalex.org/W7127645802","doi":"https://doi.org/10.1145/3769002.3769979","title":"Leveraging PyTorch for Hardware-Aware Optimization in Efficient Mixture-of-Experts Large Language Model Inference","display_name":"Leveraging PyTorch for Hardware-Aware Optimization in Efficient Mixture-of-Experts Large Language Model Inference","publication_year":2025,"publication_date":"2025-11-16","ids":{"openalex":"https://openalex.org/W7127645802","doi":"https://doi.org/10.1145/3769002.3769979"},"language":null,"primary_location":{"id":"doi:10.1145/3769002.3769979","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3769002.3769979","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference on Research in Adaptive and Convergent Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029870143","display_name":"Mu-Chi Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210098366","display_name":"Institute of Information Science, Academia Sinica","ror":"https://ror.org/00z83z196","country_code":"TW","type":"facility","lineage":["https://openalex.org/I4210098366","https://openalex.org/I84653119"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Mu-Chi Chen","raw_affiliation_strings":["Institute of Information Science, Academia Sinica, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0007-6013-4122","affiliations":[{"raw_affiliation_string":"Institute of Information Science, Academia Sinica, Taipei, Taiwan","institution_ids":["https://openalex.org/I4210098366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044258001","display_name":"Po-Hsuan Huang","orcid":"https://orcid.org/0000-0002-7458-9634"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Po-Hsuan Huang","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0000-0002-7458-9634","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102809862","display_name":"Yanwen Gai","orcid":"https://orcid.org/0009-0004-0090-4920"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Yanwen Gai","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0004-0090-4920","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125079675","display_name":"Shao-Chun Ho","orcid":null},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Shao-Chun Ho","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0000-7363-2947","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125059610","display_name":"Cheng Liang","orcid":null},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Cheng Liang","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0009-1532-3332","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103367306","display_name":"Yu\u2010Hung Kao","orcid":null},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Yu-Hung Kao","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0002-6991-8795","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yu-Kai Hung","orcid":"https://orcid.org/0009-0007-0310-4883"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Yu-Kai Hung","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0007-0310-4883","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chia-Heng Tu","orcid":"https://orcid.org/0000-0001-8967-1385"},"institutions":[{"id":"https://openalex.org/I91807558","display_name":"National Cheng Kung University","ror":"https://ror.org/01b8kcc49","country_code":"TW","type":"education","lineage":["https://openalex.org/I91807558"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Chia-Heng Tu","raw_affiliation_strings":["National Cheng Kung University, Tainan, Taiwan"],"raw_orcid":"https://orcid.org/0000-0001-8967-1385","affiliations":[{"raw_affiliation_string":"National Cheng Kung University, Tainan, Taiwan","institution_ids":["https://openalex.org/I91807558"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020028710","display_name":"Shih\u2010Hao Hung","orcid":"https://orcid.org/0000-0003-2043-2663"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Shih-Hao Hung","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0000-0003-2043-2663","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.81038923,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24709999561309814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24709999561309814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.10909999907016754,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.05090000107884407,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.8122000098228455},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7174000144004822},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.6449999809265137},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.5188000202178955},{"id":"https://openalex.org/keywords/cuda","display_name":"CUDA","score":0.5160999894142151},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.48019999265670776},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.415800005197525},{"id":"https://openalex.org/keywords/extensibility","display_name":"Extensibility","score":0.4142000079154968},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.4120999872684479}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8780999779701233},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.8122000098228455},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7174000144004822},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.6449999809265137},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5329999923706055},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.5188000202178955},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.5160999894142151},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.48019999265670776},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.415800005197525},{"id":"https://openalex.org/C32833848","wikidata":"https://www.wikidata.org/wiki/Q4115054","display_name":"Extensibility","level":2,"score":0.4142000079154968},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.4120999872684479},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4000000059604645},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.37450000643730164},{"id":"https://openalex.org/C50630238","wikidata":"https://www.wikidata.org/wiki/Q971505","display_name":"General-purpose computing on graphics processing units","level":3,"score":0.3695000112056732},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3564999997615814},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3504999876022339},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.34869998693466187},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3370000123977661},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.31470000743865967},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.30820000171661377},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.29269999265670776},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.2858000099658966},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2648000121116638},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.2632000148296356},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.26179999113082886},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.2615000009536743},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.2556999921798706}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3769002.3769979","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3769002.3769979","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Conference on Research in Adaptive and Convergent Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.4271041750907898,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W2954698171","https://openalex.org/W3081168214","https://openalex.org/W4387321091","https://openalex.org/W4388031315","https://openalex.org/W4389518760","https://openalex.org/W4408858386"],"related_works":[],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"architectures":[2],"have":[3],"become":[4],"increasingly":[5],"popular":[6],"in":[7],"large":[8],"language":[9],"models":[10],"(LLMs)":[11],"for":[12,43,52,65,75],"their":[13,48],"ability":[14],"to":[15,32,71,92,118,126,141],"provide":[16],"model":[17,81,148,157],"scalability":[18],"and":[19,28,50,82,100,136,160],"computational":[20,94],"efficiency.":[21],"While":[22],"LLM":[23,54],"inference":[24,35,67,104,164],"systems,":[25],"e.g.,":[26],"vLLM":[27],"SGLang,":[29],"are":[30],"developed":[31],"boost":[33],"the":[34,73,119],"performance,":[36],"they":[37],"rely":[38],"heavily":[39],"on":[40,112,130],"customized":[41],"optimizations":[42,77],"specific":[44],"hardware,":[45],"which":[46,154],"limits":[47],"flexibility":[49],"extensibility":[51],"private":[53],"deployments.":[55],"To":[56],"address":[57],"these":[58],"challenges,":[59],"we":[60],"present":[61],"a":[62,96,101,113,127],"PyTorch-native":[63],"framework":[64],"efficient":[66],"of":[68],"MoE-based":[69],"LLMs":[70],"eliminate":[72],"need":[74],"hardware-dependent":[76],"while":[78],"supporting":[79],"diverse":[80],"hardware":[83,115],"configurations.":[84],"Our":[85],"approach":[86],"leverages":[87],"PyTorch's":[88],"CUDA":[89],"Graphs":[90],"feature":[91],"improve":[93],"utilization,":[95],"microbenchmark-driven":[97],"performance":[98,147],"predictor,":[99],"multidimensional":[102],"parallel":[103],"engine":[105],"that":[106],"automatically":[107],"selects":[108],"optimal":[109],"parallelization":[110,151],"strategies":[111],"target":[114],"platform,":[116],"according":[117],"prediction":[120],"result.":[121],"Experimental":[122],"results":[123],"demonstrate":[124],"up":[125],"4x":[128],"speedup":[129],"both":[131],"NVIDIA":[132,137],"RTX":[133],"4090":[134],"GPUs":[135,139],"H100":[138],"compared":[140],"state-of-the-art":[142],"PyTorch":[143],"implementations.":[144],"Moreover,":[145],"our":[146],"effectively":[149],"guides":[150],"strategy":[152],"selection,":[153],"helps":[155],"reduce":[156],"tuning":[158],"overhead":[159],"enables":[161],"scalable,":[162],"high-throughput":[163],"across":[165],"varying":[166],"computing":[167],"environments.":[168]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-02-06T00:00:00"}