{"id":"https://openalex.org/W4405362385","doi":"https://doi.org/10.1109/mascots64422.2024.10786517","title":"Optimizing GPU Multiplexing for Efficient and Cost-Effective Access to Diverse Large Language Models in GPU Clusters","display_name":"Optimizing GPU Multiplexing for Efficient and Cost-Effective Access to Diverse Large Language Models in GPU Clusters","publication_year":2024,"publication_date":"2024-10-21","ids":{"openalex":"https://openalex.org/W4405362385","doi":"https://doi.org/10.1109/mascots64422.2024.10786517"},"language":"en","primary_location":{"id":"doi:10.1109/mascots64422.2024.10786517","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mascots64422.2024.10786517","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 32nd International Conference on Modeling, Analysis and Simulation of Computer and Telecommunication Systems (MASCOTS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100513379","display_name":"Yue Zhu","orcid":"https://orcid.org/0009-0009-2273-5618"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yue Zhu","raw_affiliation_strings":["IBM Research Center, Yorktown Heights,NY,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Center, Yorktown Heights,NY,USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100340740","display_name":"Wang Chen","orcid":"https://orcid.org/0000-0001-7892-3575"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen Wang","raw_affiliation_strings":["IBM Research Center, Yorktown Heights,NY,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Center, Yorktown Heights,NY,USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115429019","display_name":"Max Calman","orcid":null},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Max Calman","raw_affiliation_strings":["IBM Research Center, Yorktown Heights,NY,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Center, Yorktown Heights,NY,USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058689389","display_name":"Rina Nakazawa","orcid":null},"institutions":[{"id":"https://openalex.org/I4210145865","display_name":"IBM Research - Tokyo","ror":"https://ror.org/04915qk43","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210145865"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Rina Nakazawa","raw_affiliation_strings":["IBM Research,Tokoyo,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research,Tokoyo,Japan","institution_ids":["https://openalex.org/I4210145865"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023638218","display_name":"Eun Kyung Lee","orcid":"https://orcid.org/0000-0002-2335-4798"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eun Kyung Lee","raw_affiliation_strings":["IBM Research Center, Yorktown Heights,NY,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Center, Yorktown Heights,NY,USA","institution_ids":["https://openalex.org/I1341412227"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9573000073432922,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9573000073432922,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12702","display_name":"Brain Tumor Detection and Classification","score":0.9266999959945679,"subfield":{"id":"https://openalex.org/subfields/2808","display_name":"Neurology"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9248999953269958,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7914654016494751},{"id":"https://openalex.org/keywords/cuda","display_name":"CUDA","score":0.5872135162353516},{"id":"https://openalex.org/keywords/general-purpose-computing-on-graphics-processing-units","display_name":"General-purpose computing on graphics processing units","score":0.49560868740081787},{"id":"https://openalex.org/keywords/multiplexing","display_name":"Multiplexing","score":0.4779215455055237},{"id":"https://openalex.org/keywords/gpu-cluster","display_name":"GPU cluster","score":0.4666210412979126},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.43170398473739624},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.1604803502559662},{"id":"https://openalex.org/keywords/graphics","display_name":"Graphics","score":0.12860265374183655},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.08141976594924927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7914654016494751},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.5872135162353516},{"id":"https://openalex.org/C50630238","wikidata":"https://www.wikidata.org/wiki/Q971505","display_name":"General-purpose computing on graphics processing units","level":3,"score":0.49560868740081787},{"id":"https://openalex.org/C19275194","wikidata":"https://www.wikidata.org/wiki/Q222903","display_name":"Multiplexing","level":2,"score":0.4779215455055237},{"id":"https://openalex.org/C2781335571","wikidata":"https://www.wikidata.org/wiki/Q2633544","display_name":"GPU cluster","level":3,"score":0.4666210412979126},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.43170398473739624},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.1604803502559662},{"id":"https://openalex.org/C21442007","wikidata":"https://www.wikidata.org/wiki/Q1027879","display_name":"Graphics","level":2,"score":0.12860265374183655},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.08141976594924927}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mascots64422.2024.10786517","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mascots64422.2024.10786517","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 32nd International Conference on Modeling, Analysis and Simulation of Computer and Telecommunication Systems (MASCOTS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W3185469185","https://openalex.org/W4280642742","https://openalex.org/W4289828024","https://openalex.org/W4304192541","https://openalex.org/W4316252402","https://openalex.org/W4387321091","https://openalex.org/W4388855507","https://openalex.org/W4394007410","https://openalex.org/W4394945061","https://openalex.org/W4394969374","https://openalex.org/W6848546274","https://openalex.org/W6850625674","https://openalex.org/W6853006944","https://openalex.org/W6857524558"],"related_works":["https://openalex.org/W1963859303","https://openalex.org/W2364044215","https://openalex.org/W2389600408","https://openalex.org/W240129890","https://openalex.org/W3048701459","https://openalex.org/W2149078538","https://openalex.org/W2370314112","https://openalex.org/W1912958759","https://openalex.org/W2096672917","https://openalex.org/W2005148983"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4],"a":[5,30,72,106,128],"cornerstone":[6],"of":[7,40,53,75,99,117,135,177],"modern":[8],"artificial":[9],"intelligence":[10],"research,":[11],"gaining":[12],"popularity":[13,98,179],"and":[14,37,48,87,155,180,198],"encouraging":[15],"adoption":[16],"in":[17,50,62,111,205],"varying":[18],"domains.":[19],"The":[20,66],"burgeoning":[21],"interest":[22],"among":[23],"researchers":[24,82],"drives":[25],"the":[26,51,96,125,133,150,164,174,193,200],"need":[27],"for":[28],"building":[29],"dedicated":[31],"GPU":[32,64,85,119,141,153,194,207],"cluster":[33],"serving":[34],"an":[35],"extensive":[36],"varied":[38],"collection":[39],"LLMs,":[41],"specifically":[42],"designed":[43],"to":[44,81,115,173],"facilitate":[45],"experimental":[46],"exploration":[47],"innovation":[49],"field":[52],"AI":[54],"research.":[55],"However,":[56],"this":[57,123],"requirement":[58],"poses":[59],"significant":[60],"challenges":[61],"optimizing":[63],"multiplexing.":[65],"primary":[67],"problem":[68],"is":[69,92],"ensuring":[70],"that":[71,130,169],"wide":[73],"variety":[74],"fast-changing":[76],"LLM":[77,156],"models":[78,108],"remain":[79],"accessible":[80],"while":[83],"minimizing":[84],"costs":[86],"reducing":[88,192],"idle":[89],"time.":[90],"This":[91],"further":[93],"complicated":[94],"by":[95,196,203],"fluctuating":[97],"models;":[100],"at":[101],"any":[102],"given":[103],"time,":[104],"only":[105],"few":[107],"would":[109],"be":[110],"high":[112],"demand,":[113],"leading":[114],"underutilization":[116],"expensive":[118],"resources.":[120],"To":[121],"address":[122],"challenge,":[124],"paper":[126],"presents":[127],"methodology":[129],"involves":[131],"profiling":[132],"performance":[134,158],"various":[136,146],"LLMs":[137],"across":[138],"different":[139],"Multi-Instance":[140],"(MIG)":[142],"slice":[143],"configurations":[144],"under":[145],"loads.":[147],"By":[148],"analyzing":[149],"trade-offs":[151],"between":[152],"utilization":[154],"inference":[157],"on":[159],"vLLM":[160],"server,":[161],"we":[162],"identify":[163],"optimal":[165],"MIG":[166],"partitioning":[167],"strategies":[168],"can":[170],"dynamically":[171],"adapt":[172],"changing":[175],"landscape":[176],"model":[178],"usage":[181],"patterns":[182],"with":[183],"latency":[184],"SLA":[185],"retained":[186],"($50":[187],"\\mathrm{~ms}":[188],"/$":[189],"token),":[190],"thereby":[191],"cost":[195],"50%":[197],"enhancing":[199],"energy":[201],"efficiency":[202],"35%":[204],"our":[206],"cluster.":[208]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}