{"id":"https://openalex.org/W4406271169","doi":"https://doi.org/10.1109/tc.2025.3528125","title":"SLOpt: Serving Real-Time Inference Pipeline With Strict Latency Constraint","display_name":"SLOpt: Serving Real-Time Inference Pipeline With Strict Latency Constraint","publication_year":2025,"publication_date":"2025-01-10","ids":{"openalex":"https://openalex.org/W4406271169","doi":"https://doi.org/10.1109/tc.2025.3528125"},"language":"en","primary_location":{"id":"doi:10.1109/tc.2025.3528125","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tc.2025.3528125","pdf_url":null,"source":{"id":"https://openalex.org/S157670870","display_name":"IEEE Transactions on Computers","issn_l":"0018-9340","issn":["0018-9340","0016-9340","1557-9956","2326-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computers","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101695421","display_name":"Zhixin Zhao","orcid":"https://orcid.org/0009-0009-7865-4905"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhixin Zhao","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0009-0009-7865-4905","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102792327","display_name":"Yitao Hu","orcid":"https://orcid.org/0009-0004-0458-0900"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yitao Hu","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0009-0004-0458-0900","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Guotao Yang","orcid":"https://orcid.org/0009-0003-6392-6468"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guotao Yang","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0009-0003-6392-6468","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110267256","display_name":"Zhenhua Gong","orcid":"https://orcid.org/0009-0002-1699-8592"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziqi Gong","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0009-0002-1699-8592","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103734167","display_name":"Chen Shen","orcid":"https://orcid.org/0009-0002-1444-1272"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen Shen","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0009-0002-1444-1272","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050753595","display_name":"Laiping Zhao","orcid":"https://orcid.org/0000-0003-1967-2192"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Laiping Zhao","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0003-1967-2192","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100397205","display_name":"Wenxin Li","orcid":"https://orcid.org/0000-0001-8507-0339"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenxin Li","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0001-8507-0339","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009324770","display_name":"Xiulong Liu","orcid":"https://orcid.org/0000-0002-4746-5599"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiulong Liu","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0002-4746-5599","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041346888","display_name":"Wenyu Qu","orcid":"https://orcid.org/0000-0003-4817-5187"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenyu Qu","raw_affiliation_strings":["Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0003-4817-5187","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Advanced Networking, Department of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I162868743"],"apc_list":null,"apc_paid":null,"fwci":1.059,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.72208956,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"74","issue":"4","first_page":"1431","last_page":"1445"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9822999835014343,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9822999835014343,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.933899998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9262999892234802,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7741986513137817},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.6717830896377563},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5738705992698669},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.5455982685089111},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5081596374511719},{"id":"https://openalex.org/keywords/constraint","display_name":"Constraint (computer-aided design)","score":0.4118822515010834},{"id":"https://openalex.org/keywords/real-time-computing","display_name":"Real-time computing","score":0.3779771327972412},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3438482880592346},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.19244596362113953},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.1701779067516327},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13119018077850342}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7741986513137817},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.6717830896377563},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5738705992698669},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5455982685089111},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5081596374511719},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.4118822515010834},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3779771327972412},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3438482880592346},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.19244596362113953},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1701779067516327},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13119018077850342},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tc.2025.3528125","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tc.2025.3528125","pdf_url":null,"source":{"id":"https://openalex.org/S157670870","display_name":"IEEE Transactions on Computers","issn_l":"0018-9340","issn":["0018-9340","0016-9340","1557-9956","2326-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computers","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3404210196","display_name":null,"funder_award_id":"62202328","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6975830235","display_name":null,"funder_award_id":"2022YFB4501000","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1982063824","https://openalex.org/W2131889098","https://openalex.org/W2193145675","https://openalex.org/W2194775991","https://openalex.org/W2911181069","https://openalex.org/W2962851801","https://openalex.org/W2963122961","https://openalex.org/W2963163009","https://openalex.org/W2964081807","https://openalex.org/W2964350391","https://openalex.org/W2980994438","https://openalex.org/W2982157693","https://openalex.org/W3039010666","https://openalex.org/W3091795552","https://openalex.org/W3095488153","https://openalex.org/W3110940899","https://openalex.org/W3185772024","https://openalex.org/W3186289964","https://openalex.org/W3209166877","https://openalex.org/W4214610523","https://openalex.org/W4214690606","https://openalex.org/W4386165872","https://openalex.org/W4391559146","https://openalex.org/W4391623921","https://openalex.org/W4393904734","https://openalex.org/W6713134421","https://openalex.org/W6730956707","https://openalex.org/W6735636592","https://openalex.org/W6747759466","https://openalex.org/W6765484274","https://openalex.org/W6779103662","https://openalex.org/W6781515392","https://openalex.org/W6781728138"],"related_works":["https://openalex.org/W3037187668","https://openalex.org/W2055243143","https://openalex.org/W4234772502","https://openalex.org/W2380685755","https://openalex.org/W2252100032","https://openalex.org/W2963436428","https://openalex.org/W4400978025","https://openalex.org/W2918743509","https://openalex.org/W3128807919","https://openalex.org/W3176411177"],"abstract_inverted_index":{"The":[0],"rise":[1],"of":[2,98,127,146,191],"Machine":[3],"Learning":[4],"as":[5],"a":[6,115,124,219],"Service":[7],"(MLaaS)":[8],"has":[9],"driven":[10],"the":[11,61,70,86,91,99,144,170,176,189,192],"demand":[12],"for":[13,119],"complex":[14],"and":[15,77,94,113,132,138,149,155],"customized":[16],"real-time":[17,120],"inference":[18,30,71,100,121],"tasks,":[19],"often":[20],"requiring":[21],"cascading":[22,68],"multiple":[23],"deep":[24],"neural":[25],"network":[26],"(DNN)":[27],"models":[28],"into":[29],"pipelines.":[31],"However,":[32],"these":[33],"pipelines":[34,54,122],"pose":[35],"significant":[36],"challenges":[37],"due":[38],"to":[39,74,102,164,184,188,200,203,209],"scheduling":[40,57],"complexity,":[41],"particularly":[42],"in":[43,69,152,224],"maintaining":[44],"strict":[45,104,226],"latency":[46,105,166,179,228],"service":[47],"level":[48],"objectives":[49],"(SLOs).":[50],"Existing":[51],"systems":[52],"serve":[53],"with":[55,123],"model-independent":[56],"policies,":[58],"which":[59,141],"ignore":[60],"unique":[62],"workload":[63,96,128,136,153],"characteristics":[64],"introduced":[65],"by":[66,182,198],"model":[67],"pipeline,":[72],"leading":[73],"SLO":[75,106],"violations":[76],"resource":[78,130,156],"inefficiencies.":[79],"In":[80],"this":[81],"paper,":[82],"we":[83,111],"propose":[84],"that":[85],"serving":[87,116,196],"system":[88,117],"should":[89],"exploit":[90],"model-cascading":[92],"nature":[93],"inter-model":[95],"dependency":[97],"pipeline":[101],"ensure":[103],"cost-effectively.":[107],"Based":[108],"on":[109,218],"this,":[110],"design":[112],"implementSLOpt,":[114],"optimized":[118],"three-stage":[125],"co-design":[126],"estimation,":[129],"provisioning,":[131],"request":[133],"execution.SLOptproposes":[134],"cascade":[135,147],"estimation":[137,154],"ahead-of-time":[139],"tuning,":[140],"together":[142],"address":[143],"challenge":[145],"blocking":[148,151],"head-of-line":[150],"provisioning.SLOptfurther":[157],"implements":[158],"an":[159],"adaptive":[160],"batch":[161],"drop":[162],"policy":[163],"mitigate":[165],"amplification":[167],"issues":[168],"within":[169],"pipeline.":[171],"These":[172],"innovations":[173],"enableSLOptto":[174],"reduce":[175],"99th":[177],"percentile":[178],"(P99":[180],"latency)":[181],"1.4":[183],"2.5":[185],"times":[186],"compared":[187],"state":[190],"arts":[193],"while":[194],"lowering":[195],"costs":[197],"up":[199,208],"29%.":[201],"Moreover,":[202],"achieve":[204],"comparable":[205],"P99":[206,227],"latency,SLOptrequires":[207],"70%":[210],"less":[211],"cost":[212],"than":[213],"existing":[214],"systems.":[215],"Extensive":[216],"evaluations":[217],"64-GPU":[220],"cluster":[221],"demonstrateSLOpt\u2019s":[222],"effectiveness":[223],"meeting":[225],"SLOs":[229],"under":[230],"diverse":[231],"real-world":[232],"workloads.":[233]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}