{"id":"https://openalex.org/W7131617004","doi":"https://doi.org/10.48550/arxiv.2602.21626","title":"Multi-Layer Scheduling for MoE-Based LLM Reasoning","display_name":"Multi-Layer Scheduling for MoE-Based LLM Reasoning","publication_year":2026,"publication_date":"2026-02-25","ids":{"openalex":"https://openalex.org/W7131617004","doi":"https://doi.org/10.48550/arxiv.2602.21626"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.21626","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103133425","display_name":"Yifan Sun","orcid":"https://orcid.org/0009-0009-0339-9110"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004557184","display_name":"Gholamreza Haffar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haffari, Gholamreza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123430472","display_name":"Minxian Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Minxian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126921519","display_name":"Rajkumar Buyya","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Buyya, Rajkumar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5083902835","display_name":"Adel N. Toosi","orcid":"https://orcid.org/0000-0001-5655-5337"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Toosi, Adel N.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.15386433,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.25099998712539673,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.25099998712539673,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.14839999377727509,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.09459999948740005,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6564000248908997},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5690000057220459},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.4934000074863434},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.4869000017642975},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.447299987077713},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.43810001015663147},{"id":"https://openalex.org/keywords/inference-engine","display_name":"Inference engine","score":0.42419999837875366},{"id":"https://openalex.org/keywords/expert-system","display_name":"Expert system","score":0.31619998812675476},{"id":"https://openalex.org/keywords/two-level-scheduling","display_name":"Two-level scheduling","score":0.31529998779296875}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8428999781608582},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6564000248908997},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5690000057220459},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5601000189781189},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.4934000074863434},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.4869000017642975},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.447299987077713},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.43810001015663147},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.42419999837875366},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C119948110","wikidata":"https://www.wikidata.org/wiki/Q7858726","display_name":"Two-level scheduling","level":4,"score":0.31529998779296875},{"id":"https://openalex.org/C107568181","wikidata":"https://www.wikidata.org/wiki/Q5319000","display_name":"Dynamic priority scheduling","level":3,"score":0.3107999861240387},{"id":"https://openalex.org/C31689143","wikidata":"https://www.wikidata.org/wiki/Q733809","display_name":"Fair-share scheduling","level":3,"score":0.2890999913215637},{"id":"https://openalex.org/C138959212","wikidata":"https://www.wikidata.org/wiki/Q1806783","display_name":"Load balancing (electrical power)","level":3,"score":0.28859999775886536},{"id":"https://openalex.org/C122141398","wikidata":"https://www.wikidata.org/wiki/Q5456330","display_name":"Fixed-priority pre-emptive scheduling","level":5,"score":0.2874999940395355},{"id":"https://openalex.org/C112866106","wikidata":"https://www.wikidata.org/wiki/Q267053","display_name":"Lottery scheduling","level":5,"score":0.287200003862381},{"id":"https://openalex.org/C115874739","wikidata":"https://www.wikidata.org/wiki/Q825377","display_name":"Critical path method","level":2,"score":0.28459998965263367},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.28380000591278076},{"id":"https://openalex.org/C175893541","wikidata":"https://www.wikidata.org/wiki/Q1196582","display_name":"Round-robin scheduling","level":4,"score":0.28299999237060547},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.28200000524520874},{"id":"https://openalex.org/C19012869","wikidata":"https://www.wikidata.org/wiki/Q578372","display_name":"Response time","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C2779808786","wikidata":"https://www.wikidata.org/wiki/Q6664603","display_name":"Locality","level":2,"score":0.27799999713897705},{"id":"https://openalex.org/C127456818","wikidata":"https://www.wikidata.org/wiki/Q238879","display_name":"Rate-monotonic scheduling","level":4,"score":0.2660999894142151},{"id":"https://openalex.org/C129916263","wikidata":"https://www.wikidata.org/wiki/Q1141183","display_name":"Backward chaining","level":4,"score":0.2644999921321869}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.21626","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.21626","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.21626","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.21626","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","score":0.5400416851043701,"display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4,85],"achieved":[5],"remarkable":[6],"success":[7],"across":[8],"a":[9,21,102],"wide":[10],"range":[11],"of":[12],"tasks,":[13],"but":[14],"serving":[15],"them":[16],"efficiently":[17],"at":[18,46,53,114],"scale":[19],"remains":[20],"critical":[22],"challenge":[23],"due":[24],"to":[25,62,135,164,185,218],"their":[26],"substantial":[27],"computational":[28],"and":[29,50,67,76,96,119,132,138,161,179,188,228],"latency":[30,227],"demands.":[31],"While":[32],"most":[33],"existing":[34],"inference":[35,213],"frameworks":[36],"rely":[37],"on":[38,175],"simple":[39],"scheduling":[40,91,104,113],"strategies":[41,149],"such":[42,72,128],"as":[43,73,129],"First-Come-First-Serve":[44],"(FCFS)":[45],"the":[47,54,122,142,153,170,211],"engine":[48,143],"level":[49],"Round-Robin":[51],"(RR)":[52],"scheduler":[55],"or":[56],"coordinator":[57],"level,":[58,124,144,172],"they":[59],"often":[60],"fail":[61],"fully":[63],"utilize":[64],"system":[65],"resources":[66],"may":[68],"suffer":[69],"from":[70,93,195],"issues":[71],"head-of-line":[74],"blocking":[75],"load":[77,187],"imbalance.":[78],"Recent":[79],"advances":[80],"in":[81,90,221,231],"Mixture-of-Experts":[82],"(MoE)":[83],"models":[84],"also":[86],"introduced":[87],"new":[88],"challenges":[89],"arising":[92],"expert":[94,171,177,183],"parallelism":[95],"routing":[97,190],"complexity.":[98],"This":[99],"research":[100],"proposes":[101],"multi-layer":[103],"framework":[105,214],"tailored":[106],"for":[107,152],"MoE-based":[108],"LLM":[109],"serving.":[110],"It":[111],"targets":[112],"three":[115],"levels:":[116],"request-level,":[117],"enginelevel,":[118],"expert-level.":[120],"At":[121,141,169],"request":[123],"we":[125,145,173],"explore":[126],"algorithms":[127],"Shortest-Job-First":[130],"(SJF)":[131],"priority-aware":[133],"aging":[134],"improve":[136,189],"throughput":[137],"reduce":[139],"latency.":[140,234],"design":[146],"load-aware":[147],"dispatching":[148],"that":[150,206],"account":[151],"current":[154],"prefix":[155],"token":[156],"load,":[157],"KV":[158],"cache":[159],"utilization,":[160],"user":[162],"stickiness":[163],"achieve":[165],"better":[166],"resource":[167],"matching.":[168],"focus":[174],"alleviating":[176],"hotspots":[178],"strategically":[180],"placing":[181],"inter-layer":[182],"dependencies":[184],"balance":[186],"efficiency.":[191],"Extensive":[192],"experimental":[193],"results":[194],"more":[196],"than":[197],"100":[198],"experiments":[199],"conducted":[200],"under":[201],"diverse":[202],"workload":[203],"distributions":[204],"show":[205],"our":[207],"approach":[208],"consistently":[209],"outperforms":[210],"state-of-theart":[212],"vLLM,":[215],"achieving":[216],"up":[217],"17.8%":[219],"reduction":[220,230],"Time":[222],"To":[223],"First":[224],"Token":[225],"(TTFT)":[226],"13.3%":[229],"Time-Per-Output-Token":[232],"(TPOT)":[233]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-02-27T00:00:00"}