{"id":"https://openalex.org/W4401386998","doi":"https://doi.org/10.1109/tcss.2024.3423749","title":"SLoB: Suboptimal Load Balancing Scheduling in Local Heterogeneous GPU Clusters for Large Language Model Inference","display_name":"SLoB: Suboptimal Load Balancing Scheduling in Local Heterogeneous GPU Clusters for Large Language Model Inference","publication_year":2024,"publication_date":"2024-08-07","ids":{"openalex":"https://openalex.org/W4401386998","doi":"https://doi.org/10.1109/tcss.2024.3423749"},"language":"en","primary_location":{"id":"doi:10.1109/tcss.2024.3423749","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcss.2024.3423749","pdf_url":null,"source":{"id":"https://openalex.org/S2490693980","display_name":"IEEE Transactions on Computational Social Systems","issn_l":"2329-924X","issn":["2329-924X","2373-7476"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computational Social Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023320058","display_name":"Peiwen Jiang","orcid":"https://orcid.org/0009-0005-3787-5681"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peiwen Jiang","raw_affiliation_strings":["School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0005-3787-5681","affiliations":[{"raw_affiliation_string":"School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076251475","display_name":"Haoxin Wang","orcid":"https://orcid.org/0009-0000-9360-6084"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoxin Wang","raw_affiliation_strings":["School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0000-9360-6084","affiliations":[{"raw_affiliation_string":"School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010791300","display_name":"Zinuo Cai","orcid":"https://orcid.org/0000-0001-9373-8474"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zinuo Cai","raw_affiliation_strings":["School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-9373-8474","affiliations":[{"raw_affiliation_string":"School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113904056","display_name":"L J Gao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lintao Gao","raw_affiliation_strings":["Shanghai Aerospace System Engineering Institute, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0006-1579-2778","affiliations":[{"raw_affiliation_string":"Shanghai Aerospace System Engineering Institute, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022385009","display_name":"Weishan Zhang","orcid":"https://orcid.org/0000-0001-9800-1068"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I4210162190","display_name":"China University of Petroleum, East China","ror":"https://ror.org/05gbn2817","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210162190"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weishan Zhang","raw_affiliation_strings":["Qingdao Institute of Software, College of Computer Science and Technology, China University of Petroleum (East China), Qingdao, China"],"raw_orcid":"https://orcid.org/0000-0001-9800-1068","affiliations":[{"raw_affiliation_string":"Qingdao Institute of Software, College of Computer Science and Technology, China University of Petroleum (East China), Qingdao, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I4210162190"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036789442","display_name":"Ruhui Ma","orcid":"https://orcid.org/0000-0001-9592-8490"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruhui Ma","raw_affiliation_strings":["School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-9592-8490","affiliations":[{"raw_affiliation_string":"School of Electronic Information and Electrical Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055675863","display_name":"Xiaokang Zhou","orcid":"https://orcid.org/0000-0003-3488-4679"},"institutions":[{"id":"https://openalex.org/I56624758","display_name":"Kansai University","ror":"https://ror.org/03xg1f311","country_code":"JP","type":"education","lineage":["https://openalex.org/I56624758"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Xiaokang Zhou","raw_affiliation_strings":["Business Data Science, Kansai University, Osaka, Japan"],"raw_orcid":"https://orcid.org/0000-0003-3488-4679","affiliations":[{"raw_affiliation_string":"Business Data Science, Kansai University, Osaka, Japan","institution_ids":["https://openalex.org/I56624758"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.9209,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.87459603,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"11","issue":"6","first_page":"7941","last_page":"7951"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9693999886512756,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9693999886512756,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9054999947547913,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.683856725692749},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6763404011726379},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6524937748908997},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5027732849121094},{"id":"https://openalex.org/keywords/processor-scheduling","display_name":"Processor scheduling","score":0.4661402106285095},{"id":"https://openalex.org/keywords/gpu-cluster","display_name":"GPU cluster","score":0.4306427836418152},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.41832324862480164},{"id":"https://openalex.org/keywords/cuda","display_name":"CUDA","score":0.20113858580589294},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.19734400510787964},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.17330363392829895},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.1146135926246643}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.683856725692749},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6763404011726379},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6524937748908997},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5027732849121094},{"id":"https://openalex.org/C2984822820","wikidata":"https://www.wikidata.org/wiki/Q1123036","display_name":"Processor scheduling","level":3,"score":0.4661402106285095},{"id":"https://openalex.org/C2781335571","wikidata":"https://www.wikidata.org/wiki/Q2633544","display_name":"GPU cluster","level":3,"score":0.4306427836418152},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.41832324862480164},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.20113858580589294},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.19734400510787964},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.17330363392829895},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.1146135926246643},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcss.2024.3423749","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcss.2024.3423749","pdf_url":null,"source":{"id":"https://openalex.org/S2490693980","display_name":"IEEE Transactions on Computational Social Systems","issn_l":"2329-924X","issn":["2329-924X","2373-7476"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computational Social Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1864093853","display_name":"Research on Causality-Inspired Deep Learning Model for Anomaly Detection with Imbalanced Data","funder_award_id":"23K11064","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"}],"funders":[{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W1537809898","https://openalex.org/W2108196201","https://openalex.org/W2154703456","https://openalex.org/W2161138645","https://openalex.org/W2587003293","https://openalex.org/W2768656470","https://openalex.org/W2797405679","https://openalex.org/W2803367729","https://openalex.org/W2973727699","https://openalex.org/W3001279689","https://openalex.org/W3093606573","https://openalex.org/W3094502228","https://openalex.org/W3096484587","https://openalex.org/W3097904259","https://openalex.org/W3168867926","https://openalex.org/W3186289964","https://openalex.org/W4249565289","https://openalex.org/W4285247752","https://openalex.org/W4288028629","https://openalex.org/W4292779060","https://openalex.org/W4312380686","https://openalex.org/W4321636575","https://openalex.org/W4362515116","https://openalex.org/W4372263604","https://openalex.org/W4372267076","https://openalex.org/W4376167141","https://openalex.org/W4378696997","https://openalex.org/W4379924764","https://openalex.org/W4386385424","https://openalex.org/W4387321109","https://openalex.org/W4388284731","https://openalex.org/W4390075082","https://openalex.org/W4391272922","https://openalex.org/W6767997687","https://openalex.org/W6770820644","https://openalex.org/W6772383348","https://openalex.org/W6778883912","https://openalex.org/W6779103662","https://openalex.org/W6784333009","https://openalex.org/W6785197036","https://openalex.org/W6796581206","https://openalex.org/W6851775633","https://openalex.org/W6852643386","https://openalex.org/W6852691820","https://openalex.org/W6853325866","https://openalex.org/W6861569072"],"related_works":["https://openalex.org/W1966243865","https://openalex.org/W1582436825","https://openalex.org/W2128410848","https://openalex.org/W3048263112","https://openalex.org/W4386430090","https://openalex.org/W2102390841","https://openalex.org/W2118368532","https://openalex.org/W2126232624","https://openalex.org/W2434525066","https://openalex.org/W3140149227"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"are":[4,45],"becoming":[5],"powerful":[6],"engines":[7],"for":[8,77,88],"social":[9],"productivity":[10],"in":[11,109,159],"the":[12,48,66,89,144,154,164,181,186,198],"manufacturing":[13],"lifecycle.":[14],"Existing":[15],"application-level":[16],"LLMs":[17,105,122],"inference":[18,39,78,106,132],"services":[19,44],"focus":[20],"on":[21,163],"large":[22],"datacenter":[23],"and":[24,38,80,127],"small":[25],"edge":[26],"intelligence":[27],"(EI)":[28],"scenarios,":[29],"adopting":[30],"iteration-level":[31],"batch":[32],"schedulers":[33],"to":[34,120,147,195],"solve":[35],"resource":[36,79,129],"utilization":[37,130],"speed":[40],"problems.":[41],"However,":[42],"these":[43],"incompatible":[46],"with":[47,59,94,131,143,185,197],"scene":[49,73],"of":[50,72,92,125,156,179],"medium-sized":[51,110],"local":[52,111],"heterogeneous":[53,112],"graphics":[54],"processing":[55],"unit":[56],"(GPU)":[57],"clusters":[58],"specific":[60],"patterns,":[61],"whose":[62],"scale":[63],"is":[64],"between":[65],"two":[67],"aforementioned":[68],"scenarios.":[69],"This":[70],"type":[71],"proposes":[74],"tradeoff":[75],"problems":[76,87],"speed,":[81],"as":[82,84],"well":[83],"user":[85],"satisfaction":[86,182],"semisparse":[90,135],"frequency":[91],"queries":[93],"streaming":[95,160],"responses.":[96,161],"We":[97],"propose":[98],"suboptimal":[99],"load":[100],"balancing":[101],"(SLoB),":[102],"a":[103,117,139,151,176],"distributed":[104],"service":[107],"scheduler":[108,142],"GPU":[113],"clusters.":[114],"SLoB":[115,174],"leverages":[116],"multilevel":[118],"adapter":[119],"accommodate":[121],"usage":[123],"patterns":[124],"scenes":[126],"balance":[128],"efficiency.":[133],"For":[134],"problems,":[136],"it":[137],"adopts":[138],"mixed-priority":[140],"pipeline":[141],"least-padding":[145],"principle":[146],"improve":[148],"users\u2019":[149],"satisfaction,":[150],"metric":[152,183],"considering":[153],"weights":[155],"different":[157],"tokens":[158],"Based":[162],"system":[165],"prototype,":[166],"our":[167],"experiments":[168],"under":[169],"simulated":[170],"workloads":[171],"demonstrate":[172],"that":[173],"gains":[175],"maximum":[177],"improvement":[178],"29.4$\\times$under":[180],"compared":[184],"traditional":[187],"run-to-completion":[188],"scheduling":[189],"solution":[190,201],"while":[191],"improving":[192],"by":[193],"up":[194],"3.0$\\times$compared":[196],"state-of-the-art":[199],"(SOTA)":[200],"Orca.":[202]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}