{"id":"https://openalex.org/W4390660333","doi":"https://doi.org/10.1109/tcc.2024.3350561","title":"BatOpt: Optimizing GPU-Based Deep Learning Inference Using Dynamic Batch Processing","display_name":"BatOpt: Optimizing GPU-Based Deep Learning Inference Using Dynamic Batch Processing","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4390660333","doi":"https://doi.org/10.1109/tcc.2024.3350561"},"language":"en","primary_location":{"id":"doi:10.1109/tcc.2024.3350561","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2024.3350561","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100705648","display_name":"Deyu Zhang","orcid":"https://orcid.org/0000-0002-5676-1285"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Deyu Zhang","raw_affiliation_strings":["School of Computer Science and Engineering, Central South University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-5676-1285","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Central South University, Changsha, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053686418","display_name":"Yunzhen Luo","orcid":"https://orcid.org/0000-0002-9359-3401"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunzhen Luo","raw_affiliation_strings":["College of Computer Science and Electronic Engineering, Hunan University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0001-6143-2315","affiliations":[{"raw_affiliation_string":"College of Computer Science and Electronic Engineering, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109672253","display_name":"Yaobo Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaobo Wang","raw_affiliation_strings":["School of Computer Science and Engineering, Central South University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-6034-8012","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Central South University, Changsha, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077488582","display_name":"Xiaoyan Kui","orcid":"https://orcid.org/0000-0002-9957-7867"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyan Kui","raw_affiliation_strings":["School of Computer Science and Engineering, Central South University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-9957-7867","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Central South University, Changsha, China","institution_ids":["https://openalex.org/I139660479"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015419107","display_name":"Ju Ren","orcid":"https://orcid.org/0000-0003-2782-183X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ju Ren","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-2782-183X","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.635,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.83677819,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"12","issue":"1","first_page":"174","last_page":"185"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8212090730667114},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.684500515460968},{"id":"https://openalex.org/keywords/general-purpose-computing-on-graphics-processing-units","display_name":"General-purpose computing on graphics processing units","score":0.6166412234306335},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5766423344612122},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5327297449111938},{"id":"https://openalex.org/keywords/graphics-processing-unit","display_name":"Graphics processing unit","score":0.5064479112625122},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4937632977962494},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.4834628701210022},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.18216994404792786},{"id":"https://openalex.org/keywords/graphics","display_name":"Graphics","score":0.1593705415725708},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.11583298444747925}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8212090730667114},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.684500515460968},{"id":"https://openalex.org/C50630238","wikidata":"https://www.wikidata.org/wiki/Q971505","display_name":"General-purpose computing on graphics processing units","level":3,"score":0.6166412234306335},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5766423344612122},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5327297449111938},{"id":"https://openalex.org/C2779851693","wikidata":"https://www.wikidata.org/wiki/Q183484","display_name":"Graphics processing unit","level":2,"score":0.5064479112625122},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4937632977962494},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.4834628701210022},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.18216994404792786},{"id":"https://openalex.org/C21442007","wikidata":"https://www.wikidata.org/wiki/Q1027879","display_name":"Graphics","level":2,"score":0.1593705415725708},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.11583298444747925}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcc.2024.3350561","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcc.2024.3350561","pdf_url":null,"source":{"id":"https://openalex.org/S2492498579","display_name":"IEEE Transactions on Cloud Computing","issn_l":"2168-7161","issn":["2168-7161","2372-0018"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cloud Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5099999904632568,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G1921880155","display_name":null,"funder_award_id":"62177047","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3327894705","display_name":null,"funder_award_id":"2021JC0004","funder_id":"https://openalex.org/F4320322843","funder_display_name":"Natural Science Foundation of\u00a0Hunan Province"},{"id":"https://openalex.org/G5507408182","display_name":null,"funder_award_id":"62172439","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6404191202","display_name":null,"funder_award_id":"U22A2034","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7287995061","display_name":null,"funder_award_id":"2023QYJC020","funder_id":"https://openalex.org/F4320321514","funder_display_name":"Central South University"},{"id":"https://openalex.org/G7644884317","display_name":null,"funder_award_id":"2023CXQD061","funder_id":"https://openalex.org/F4320321514","funder_display_name":"Central South University"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321514","display_name":"Central South University","ror":"https://ror.org/00f1zfq44"},{"id":"https://openalex.org/F4320322843","display_name":"Natural Science Foundation of\u00a0Hunan Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W147194447","https://openalex.org/W1575935080","https://openalex.org/W2012309514","https://openalex.org/W2027753527","https://openalex.org/W2046481148","https://openalex.org/W2087866311","https://openalex.org/W2091571552","https://openalex.org/W2123029116","https://openalex.org/W2577255746","https://openalex.org/W2790412025","https://openalex.org/W2792220137","https://openalex.org/W2885819483","https://openalex.org/W2893813411","https://openalex.org/W2903650079","https://openalex.org/W2963884515","https://openalex.org/W2963918968","https://openalex.org/W2965289829","https://openalex.org/W2970971581","https://openalex.org/W2980789769","https://openalex.org/W2986420675","https://openalex.org/W2996600026","https://openalex.org/W3108212103","https://openalex.org/W3117103847","https://openalex.org/W3168997536","https://openalex.org/W4206749691","https://openalex.org/W4282970339","https://openalex.org/W4289276774","https://openalex.org/W6605943672","https://openalex.org/W6731862131","https://openalex.org/W6756580189","https://openalex.org/W6762488536"],"related_works":["https://openalex.org/W2151046618","https://openalex.org/W1972148443","https://openalex.org/W1969233021","https://openalex.org/W2167646277","https://openalex.org/W2027443981","https://openalex.org/W2388314963","https://openalex.org/W3158047141","https://openalex.org/W2063573318","https://openalex.org/W1656096860","https://openalex.org/W2360624069"],"abstract_inverted_index":{"Deep":[0],"learning":[1],"(DL)":[2],"has":[3],"been":[4],"applied":[5],"in":[6,16,96,129,149,186],"billions":[7],"of":[8,28,34,58,68,116,188,208,214],"mobile":[9,29],"devices":[10],"due":[11],"to":[12,39,42,54,70,84,123,139,179,192],"its":[13],"astonishing":[14],"performance":[15,184],"image,":[17],"text,":[18],"and":[19,92,145,164,181,194],"audio":[20],"processing.":[21],"However,":[22],"limited":[23],"by":[24,153],"the":[25,56,63,72,104,114,125,142,155,212,215],"computing":[26,66],"capability":[27],"devices,":[30],"a":[31,77,86,109],"large":[32],"amount":[33],"DL":[35,97,105,174],"inference":[36,98,106],"tasks":[37],"need":[38],"be":[40],"offloaded":[41],"edge":[43],"or":[44],"cloud":[45],"servers,":[46],"which":[47,120],"makes":[48],"powerful":[49],"GPU":[50,69,93,146,171,201],"servers":[51],"are":[52],"struggling":[53],"ensure":[55],"quality":[57],"service(QoS).":[59],"To":[60],"better":[61],"utilize":[62],"highly":[64],"parallel":[65],"architecture":[67],"improve":[71],"QoS,":[73],"we":[74,134],"propose":[75,135],"BatOpt,":[76],"framework":[78],"that":[79,207],"uses":[80],"dynamic":[81],"batch":[82],"processing":[83],"strike":[85],"good":[87],"balance":[88],"between":[89],"service":[90,107,126,143,189,217],"latency":[91,127,144],"memory":[94,147,202],"usage":[95,148,203],"services.":[99],"Specifically,":[100],"BatOpt":[101,161,176],"innovatively":[102],"models":[103],"as":[108],"<inline-formula><tex-math":[110],"notation=\"LaTeX\">$M/G(a,b)/1/N$</tex-math></inline-formula>":[111],"queue,":[112],"with":[113],"consideration":[115],"stochastic":[117],"task":[118],"arrivals,":[119],"enables":[121],"it":[122,166],"predict":[124],"accurately":[128],"different":[130,150],"system":[131,151],"states.":[132],"Furthermore,":[133],"an":[136,168],"optimization":[137],"algorithm":[138],"trade":[140],"off":[141],"states":[152],"analyzing":[154],"queueing":[156],"model.":[157],"We":[158],"have":[159],"implemented":[160],"on":[162,167,211],"Pytorch":[163],"evaluated":[165],"RTX":[169],"2080":[170],"using":[172],"real":[173],"models.":[175],"brings":[177],"up":[178],"31x":[180],"4.3x":[182],"times":[183],"boost":[185],"terms":[187],"latency,":[190],"compared":[191],"single-input":[193],"fixed-batch-size":[195],"strategies,":[196],"respectively.":[197],"And":[198],"BatOpt's":[199],"maximum":[200],"is":[204],"only":[205],"0.3x":[206],"greedy-dynamic-batch-size":[209],"strategy":[210],"premise":[213],"same":[216],"latency.":[218]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}