{"id":"https://openalex.org/W7169805433","doi":"https://doi.org/10.1145/3788550.3794877","title":"ASTL: An Adaptive Serving Stack for LLMs to Balance Utilization and Tail Latency","display_name":"ASTL: An Adaptive Serving Stack for LLMs to Balance Utilization and Tail Latency","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7169805433","doi":"https://doi.org/10.1145/3788550.3794877"},"language":null,"primary_location":{"id":"doi:10.1145/3788550.3794877","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3788550.3794877","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Software Engineering for Adaptive and Self-Managing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3788550.3794877","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023376863","display_name":"Farhoud Jafari Kaleibar","orcid":"https://orcid.org/0000-0001-9726-5408"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Farhoud Jafari Kaleibar","raw_affiliation_strings":["York University, Toronto, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0001-9726-5408","affiliations":[{"raw_affiliation_string":"York University, Toronto, ON, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051000791","display_name":"Marin Litoiu","orcid":"https://orcid.org/0000-0003-0383-920X"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Marin Litoiu","raw_affiliation_strings":["York University, Toronto, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0383-920X","affiliations":[{"raw_affiliation_string":"York University, Toronto, ON, Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I192455969"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"142","last_page":"147"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.7678999900817871},{"id":"https://openalex.org/keywords/stack","display_name":"Stack (abstract data type)","score":0.6499000191688538},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.5278000235557556},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.482699990272522},{"id":"https://openalex.org/keywords/parallelism","display_name":"Parallelism (grammar)","score":0.41819998621940613},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.41499999165534973},{"id":"https://openalex.org/keywords/central-processing-unit","display_name":"Central processing unit","score":0.4088999927043915}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.786899983882904},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.7678999900817871},{"id":"https://openalex.org/C9395851","wikidata":"https://www.wikidata.org/wiki/Q177929","display_name":"Stack (abstract data type)","level":2,"score":0.6499000191688538},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.5278000235557556},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.482699990272522},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.447299987077713},{"id":"https://openalex.org/C2781172179","wikidata":"https://www.wikidata.org/wiki/Q853109","display_name":"Parallelism (grammar)","level":2,"score":0.41819998621940613},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.41499999165534973},{"id":"https://openalex.org/C49154492","wikidata":"https://www.wikidata.org/wiki/Q5300","display_name":"Central processing unit","level":2,"score":0.4088999927043915},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4068000018596649},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.38769999146461487},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.3531999886035919},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.31139999628067017},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2883000075817108},{"id":"https://openalex.org/C197115733","wikidata":"https://www.wikidata.org/wiki/Q1003136","display_name":"Forcing (mathematics)","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C38601921","wikidata":"https://www.wikidata.org/wiki/Q1757693","display_name":"Protocol stack","level":3,"score":0.26249998807907104},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C168031717","wikidata":"https://www.wikidata.org/wiki/Q1530280","display_name":"Balance (ability)","level":2,"score":0.2533000111579895}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3788550.3794877","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3788550.3794877","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Software Engineering for Adaptive and Self-Managing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3788550.3794877","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3788550.3794877","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Software Engineering for Adaptive and Self-Managing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W2963748441","https://openalex.org/W4387321091","https://openalex.org/W4387701050","https://openalex.org/W4412889466","https://openalex.org/W4413360549","https://openalex.org/W7133227460"],"related_works":[],"abstract_inverted_index":{"We":[0],"present":[1],"ASTL,":[2],"a":[3,58,116,122],"practical":[4],"serving":[5],"stack":[6],"that":[7,41,62],"adapts":[8],"runtime":[9],"parallelism":[10,44],"to":[11,46,92],"each":[12],"request\u2019s":[13],"predicted":[14],"output":[15,64],"length,":[16,76],"improving":[17],"throughput":[18],"and":[19,27,56,98,113,121],"tail":[20],"latency":[21],"for":[22,51,106],"LLM":[23],"inference":[24],"on":[25,32,115],"CPU":[26,117],"GPU.":[28],"The":[29],"design":[30],"rests":[31],"two":[33],"ingredients:":[34],"(i)":[35],"an":[36],"offline":[37],"knee":[38,105],"identification":[39],"step":[40],"profiles":[42],"sequence-level":[43],"(NP)":[45],"find":[47],"regime-specific":[48],"operating":[49],"points":[50],"short":[52],"vs.":[53],"long":[54],"generations;":[55],"(ii)":[57],"lightweight":[59],"XGBoost":[60],"predictor":[61],"estimates":[63],"tokens":[65],"at":[66],"admission":[67],"time":[68],"using":[69],"only":[70],"proxy,":[71],"admission-available":[72],"cues":[73],"(e.g.,":[74],"input":[75],"presence":[77],"of":[78],"code":[79],"fences/backticks,":[80],"punctuation":[81],"patterns,":[82],"temperature/top-p)\u2014without":[83],"explicit":[84],"domain":[85],"labels.":[86],"At":[87],"runtime,":[88],"ASTL":[89,130],"routes":[90],"requests":[91],"per-NP":[93],"lanes":[94],"with":[95],"continuous":[96],"batching":[97],"length-homogeneous":[99],"micro-batches,":[100],"running":[101],"just":[102],"below":[103],"the":[104,107],"inferred":[108],"regime.":[109],"Implemented":[110],"atop":[111],"llama.cpp":[112],"evaluated":[114],"m5.4xlarge":[118],"(Llama-2-7B":[119],"Q4)":[120,126],"single-GPU":[123],"A10":[124],"(Llama-2-13B":[125],"under":[127],"mixed/short":[128],"workloads,":[129],"reduces":[131],"p95/p99":[132],"tails":[133],"while":[134],"preserving":[135],"near-peak":[136],"throughput.":[137]},"counts_by_year":[],"updated_date":"2026-07-21T05:56:28.042755","created_date":"2026-07-21T00:00:00"}