{"id":"https://openalex.org/W7165138716","doi":"https://doi.org/10.48550/arxiv.2606.18431","title":"Beyond Prediction: Tail-Aware Scheduling for LLM Inference","display_name":"Beyond Prediction: Tail-Aware Scheduling for LLM Inference","publication_year":2026,"publication_date":"2026-06-16","ids":{"openalex":"https://openalex.org/W7165138716","doi":"https://doi.org/10.48550/arxiv.2606.18431"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.18431","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18431","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.18431","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138879768","display_name":"Yueying Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yueying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055208000","display_name":"Yanlun Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yuanfan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062587013","display_name":"J. Chen","orcid":"https://orcid.org/0009-0003-1305-9159"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jiayang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045708568","display_name":"Esha Choukse","orcid":"https://orcid.org/0000-0003-0371-5522"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choukse, Esha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037793114","display_name":"Haoran Qiu","orcid":"https://orcid.org/0000-0002-8071-1130"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Haoran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138962456","display_name":"G. Edward Suh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Suh, G. Edward","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036529548","display_name":"Rodrigo Fonseca","orcid":"https://orcid.org/0000-0001-9662-2661"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fonseca, Rodrigo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014560251","display_name":"Ziv Scully","orcid":"https://orcid.org/0000-0002-8547-1068"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scully, Ziv","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5035704218","display_name":"Udit Gupta","orcid":"https://orcid.org/0000-0002-9118-0961"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gupta, Udit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.2865000069141388,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.2865000069141388,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.1876000016927719,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.17059999704360962,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.7063999772071838},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.5281000137329102},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.4699000120162964},{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.4577000141143799},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4424000084400177},{"id":"https://openalex.org/keywords/processor-scheduling","display_name":"Processor scheduling","score":0.4147999882698059},{"id":"https://openalex.org/keywords/statistical-inference","display_name":"Statistical inference","score":0.40380001068115234},{"id":"https://openalex.org/keywords/fair-share-scheduling","display_name":"Fair-share scheduling","score":0.3901999890804291}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.770799994468689},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.7063999772071838},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.5281000137329102},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.4699000120162964},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.4577000141143799},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4424000084400177},{"id":"https://openalex.org/C2984822820","wikidata":"https://www.wikidata.org/wiki/Q1123036","display_name":"Processor scheduling","level":3,"score":0.4147999882698059},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.4124000072479248},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.40380001068115234},{"id":"https://openalex.org/C31689143","wikidata":"https://www.wikidata.org/wiki/Q733809","display_name":"Fair-share scheduling","level":3,"score":0.3901999890804291},{"id":"https://openalex.org/C122141398","wikidata":"https://www.wikidata.org/wiki/Q5456330","display_name":"Fixed-priority pre-emptive scheduling","level":5,"score":0.3695000112056732},{"id":"https://openalex.org/C206952183","wikidata":"https://www.wikidata.org/wiki/Q1193100","display_name":"Preemption","level":2,"score":0.36390000581741333},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.36070001125335693},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3346000015735626},{"id":"https://openalex.org/C127456818","wikidata":"https://www.wikidata.org/wiki/Q238879","display_name":"Rate-monotonic scheduling","level":4,"score":0.3280999958515167},{"id":"https://openalex.org/C175893541","wikidata":"https://www.wikidata.org/wiki/Q1196582","display_name":"Round-robin scheduling","level":4,"score":0.32089999318122864},{"id":"https://openalex.org/C107568181","wikidata":"https://www.wikidata.org/wiki/Q5319000","display_name":"Dynamic priority scheduling","level":3,"score":0.30649998784065247},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.29840001463890076},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.2955000102519989},{"id":"https://openalex.org/C71559656","wikidata":"https://www.wikidata.org/wiki/Q671298","display_name":"Divide and conquer algorithms","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.2946999967098236},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.27619999647140503},{"id":"https://openalex.org/C119948110","wikidata":"https://www.wikidata.org/wiki/Q7858726","display_name":"Two-level scheduling","level":4,"score":0.275299996137619},{"id":"https://openalex.org/C2986587452","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical analysis","level":2,"score":0.26750001311302185},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.26600000262260437},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.18431","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18431","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.18431","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18431","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM":[0,13,152],"serving":[1],"exhibits":[2],"extreme":[3],"length":[4,79,126],"variability,":[5],"making":[6],"size-based":[7],"scheduling":[8,74,93],"difficult":[9],"in":[10,150],"practice.":[11],"Recent":[12],"schedulers":[14],"approximate":[15],"SJF/SRPT":[16],"using":[17],"predicted":[18],"decode":[19,101],"lengths":[20],"or":[21],"ranks":[22],"and":[23,31,47,94,109,128,137],"primarily":[24],"report":[25],"mean-centric":[26],"metrics":[27],"such":[28],"as":[29],"TTFT":[30,130],"TBT.":[32],"We":[33,69],"show":[34],"that":[35,60,76],"these":[36],"prediction-driven":[37],"policies":[38],"can":[39],"be":[40],"fragile":[41],"under":[42],"distribution":[43],"shifts,":[44],"bursty":[45],"arrivals,":[46],"GPU":[48],"memory":[49],"pressure,":[50],"while":[51],"offering":[52],"limited":[53],"control":[54],"over":[55],"the":[56],"tail":[57,148],"latency":[58,149],"(P90-P99)":[59],"dominates":[61],"user":[62],"experience,":[63],"even":[64],"with":[65,81,124],"perfect":[66,125],"decode-length":[67],"knowledge.":[68],"introduce":[70],"a":[71,143],"distribution-aware,":[72],"prediction-free":[73],"framework":[75],"replaces":[77],"explicit":[78],"prediction":[80],"soft":[82],"priority":[83],"boosting":[84],"driven":[85],"by":[86,117,131],"lightweight":[87],"statistical":[88],"signals.":[89],"Our":[90],"design":[91],"co-optimizes":[92],"cache-aware":[95],"preemption":[96],"to":[97,119,122],"account":[98],"for":[99,146],"memory-coupled":[100],"dynamics":[102],"across":[103,133],"workload":[104],"mixes.":[105],"Evaluated":[106],"on":[107],"production":[108],"open-source":[110],"traces,":[111],"our":[112],"method":[113],"reduces":[114,129],"P99":[115],"TTLT":[116],"up":[118],"35-50%":[120],"relative":[121],"SRPT":[123],"knowledge":[127],"34-47%":[132],"workloads,":[134],"including":[135],"reasoning-heavy":[136],"chat-heavy":[138],"tasks.":[139],"These":[140],"results":[141],"demonstrate":[142],"robust":[144],"alternative":[145],"optimizing":[147],"online":[151],"serving.":[153]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-19T00:00:00"}