{"id":"https://openalex.org/W4417512510","doi":"https://doi.org/10.48550/arxiv.2505.06481","title":"QoS-Efficient Serving of Multiple Mixture-of-Expert LLMs Using Partial Runtime Reconfiguration","display_name":"QoS-Efficient Serving of Multiple Mixture-of-Expert LLMs Using Partial Runtime Reconfiguration","publication_year":2025,"publication_date":"2025-05-10","ids":{"openalex":"https://openalex.org/W4417512510","doi":"https://doi.org/10.48550/arxiv.2505.06481"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2505.06481","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.06481","pdf_url":"https://arxiv.org/pdf/2505.06481","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.06481","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107580492","display_name":"HamidReza Imani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Imani, HamidReza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029525752","display_name":"Jiaxin Peng","orcid":"https://orcid.org/0000-0002-5759-6904"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Jiaxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120861646","display_name":"Peiman Mohseni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mohseni, Peiman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005083867","display_name":"Abdolah Amirany","orcid":"https://orcid.org/0000-0003-0298-6945"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Amirany, Abdolah","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5001914825","display_name":"Tarek El\u2010Ghazawi","orcid":"https://orcid.org/0000-0001-9687-7939"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"El-Ghazawi, Tarek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.1695999950170517,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.1695999950170517,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.06859999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03629999980330467,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7541000247001648},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.565500020980835},{"id":"https://openalex.org/keywords/control-reconfiguration","display_name":"Control reconfiguration","score":0.5600000023841858},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.557200014591217},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.5038999915122986},{"id":"https://openalex.org/keywords/virtualization","display_name":"Virtualization","score":0.46650001406669617},{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.42160001397132874},{"id":"https://openalex.org/keywords/shared-resource","display_name":"Shared resource","score":0.3935000002384186},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.3257000148296356},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.3253999948501587}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8327999711036682},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7541000247001648},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.565500020980835},{"id":"https://openalex.org/C119701452","wikidata":"https://www.wikidata.org/wiki/Q5165881","display_name":"Control reconfiguration","level":2,"score":0.5600000023841858},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.557200014591217},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5077999830245972},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.5038999915122986},{"id":"https://openalex.org/C513985346","wikidata":"https://www.wikidata.org/wiki/Q270471","display_name":"Virtualization","level":3,"score":0.46650001406669617},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.42160001397132874},{"id":"https://openalex.org/C51332947","wikidata":"https://www.wikidata.org/wiki/Q1172305","display_name":"Shared resource","level":2,"score":0.3935000002384186},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.367000013589859},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.3398999869823456},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.3257000148296356},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.3253999948501587},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3221000134944916},{"id":"https://openalex.org/C176553487","wikidata":"https://www.wikidata.org/wiki/Q7855819","display_name":"Turnaround time","level":2,"score":0.3156999945640564},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.30090001225471497},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.29840001463890076},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C2779585090","wikidata":"https://www.wikidata.org/wiki/Q3457762","display_name":"Resilience (materials science)","level":2,"score":0.2833999991416931},{"id":"https://openalex.org/C81293917","wikidata":"https://www.wikidata.org/wiki/Q4189534","display_name":"System deployment","level":3,"score":0.2782999873161316},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.27300000190734863},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.2676999866962433},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C38725249","wikidata":"https://www.wikidata.org/wiki/Q913876","display_name":"Time-sharing","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C20136886","wikidata":"https://www.wikidata.org/wiki/Q749647","display_name":"Interoperability","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C132943942","wikidata":"https://www.wikidata.org/wiki/Q2562511","display_name":"Footprint","level":2,"score":0.2574000060558319},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.257099986076355},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.25679999589920044},{"id":"https://openalex.org/C2780615140","wikidata":"https://www.wikidata.org/wiki/Q920419","display_name":"Upgrade","level":2,"score":0.2549000084400177},{"id":"https://openalex.org/C20574231","wikidata":"https://www.wikidata.org/wiki/Q844605","display_name":"Backward compatibility","level":2,"score":0.2540999948978424},{"id":"https://openalex.org/C5119721","wikidata":"https://www.wikidata.org/wiki/Q220501","display_name":"Quality of service","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2505.06481","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.06481","pdf_url":"https://arxiv.org/pdf/2505.06481","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.06481","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.06481","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.06481","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.06481","pdf_url":"https://arxiv.org/pdf/2505.06481","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"deployment":[1],"of":[2,37,46,169],"mixture-of-experts":[3],"(MoE)":[4],"large":[5],"language":[6],"models":[7,136],"(LLMs)":[8],"presents":[9],"significant":[10],"challenges":[11,19],"due":[12],"to":[13,65,110,146,161,177],"their":[14],"high":[15],"memory":[16,69],"demands.":[17],"These":[18],"become":[20],"even":[21],"more":[22],"pronounced":[23],"in":[24,120,142,172],"multi-tenant":[25],"environments,":[26],"where":[27],"shared":[28],"resources":[29],"must":[30],"accommodate":[31],"multiple":[32,49],"models,":[33],"limiting":[34],"the":[35,44,67,165],"effectiveness":[36],"conventional":[38],"virtualization":[39],"techniques.":[40],"This":[41],"paper":[42],"addresses":[43],"problem":[45],"efficiently":[47],"serving":[48,58,111],"fine-tuned":[50],"MoE-LLMs":[51],"on":[52,124,153],"a":[53,57,97,102,112,117,125,128],"single-GPU.":[54],"We":[55],"propose":[56],"system":[59],"that":[60],"employs":[61],"\\textit{similarity-based":[62],"expert":[63],"consolidation}":[64],"reduce":[66],"overall":[68],"footprint":[70],"by":[71],"sharing":[72],"similar":[73],"experts":[74],"across":[75],"models.":[76,95],"To":[77],"ensure":[78],"output":[79,104,174],"quality,":[80],"we":[81],"introduce":[82],"\\textit{runtime":[83],"partial":[84],"reconfiguration},":[85],"dynamically":[86],"replacing":[87],"non-expert":[88],"layers":[89],"when":[90],"processing":[91],"requests":[92],"from":[93],"different":[94],"As":[96],"result,":[98],"our":[99,170],"approach":[100,171],"achieves":[101],"competitive":[103],"quality":[105,175],"while":[106,115],"maintaining":[107,173],"throughput":[108],"comparable":[109],"single":[113,129],"model":[114,158,179],"incurring":[116],"negligible":[118],"increase":[119],"time-to-first-token":[121],"(TTFT).":[122],"Experiments":[123],"server":[126],"with":[127,159],"NVIDIA":[130],"A100":[131],"GPU":[132,149],"(80GB)":[133],"using":[134],"Mixtral-8x7B":[135],"demonstrate":[137,164],"an":[138],"85\\%":[139],"average":[140],"reduction":[141],"turnaround":[143],"time":[144],"compared":[145,176],"NVIDIA's":[147],"multi-instance":[148],"(MIG).":[150],"Furthermore,":[151],"experiments":[152],"Google's":[154],"Switch":[155],"Transformer":[156],"Base-8":[157],"up":[160],"four":[162],"variants":[163],"scalability":[166],"and":[167],"resilience":[168],"other":[178],"merging":[180],"baselines,":[181],"highlighting":[182],"its":[183],"effectiveness.":[184]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}