{"id":"https://openalex.org/W4415277244","doi":"https://doi.org/10.1145/3806645.3807813","title":"STAR: Decode-Phase Rescheduling for LLM Inference","display_name":"STAR: Decode-Phase Rescheduling for LLM Inference","publication_year":2026,"publication_date":"2026-07-11","ids":{"openalex":"https://openalex.org/W4415277244","doi":"https://doi.org/10.1145/3806645.3807813"},"language":null,"primary_location":{"id":"doi:10.1145/3806645.3807813","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3806645.3807813","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3806645.3807813","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100332296","display_name":"Zhibin Wang","orcid":"https://orcid.org/0000-0003-4149-031X"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhibin Wang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0001-9204-4075","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120034746","display_name":"Zetao Hong","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zetao Hong","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0008-0780-5799","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100733661","display_name":"Xue Li","orcid":"https://orcid.org/0000-0002-0649-6006"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xue Li","raw_affiliation_strings":["Alibaba Group, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-5713-7225","affiliations":[{"raw_affiliation_string":"Alibaba Group, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zibo Wang","orcid":"https://orcid.org/0009-0009-0499-5328"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zibo Wang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0009-0499-5328","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020385539","display_name":"Shipeng Li","orcid":"https://orcid.org/0000-0002-1507-8674"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shipeng Li","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0005-7806-456X","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qingkai Meng","orcid":"https://orcid.org/0000-0001-5394-8450"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingkai Meng","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0001-5394-8450","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060395904","display_name":"Qing Wang","orcid":"https://orcid.org/0000-0003-0950-1111"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qing Wang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-5526-7154","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091007366","display_name":"Chengying Huan","orcid":"https://orcid.org/0000-0002-3154-3580"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengying Huan","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-3154-3580","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052175650","display_name":"Rong Gu","orcid":"https://orcid.org/0000-0002-1565-9997"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rong Gu","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-1565-9997","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101458852","display_name":"Sheng Zhong","orcid":"https://orcid.org/0000-0003-1694-5028"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sheng Zhong","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-6581-8730","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100751736","display_name":"Chen Tian","orcid":"https://orcid.org/0000-0003-2710-7628"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen Tian","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-2710-7628","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"125","last_page":"138"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14276","display_name":"Power Systems and Technologies","score":0.8251000046730042,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14276","display_name":"Power Systems and Technologies","score":0.8251000046730042,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.7332000136375427},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.6211000084877014},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6169999837875366},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.609000027179718},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.45010000467300415},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.35659998655319214}],"concepts":[{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.7332000136375427},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6980000138282776},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.6211000084877014},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6169999837875366},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.609000027179718},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.45100000500679016},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.45010000467300415},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.35659998655319214},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.3422999978065491},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3100999891757965},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3084000051021576},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.30640000104904175},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.29809999465942383},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.2978000044822693},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27309998869895935},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.2700999975204468},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2676999866962433},{"id":"https://openalex.org/C3017489831","wikidata":"https://www.wikidata.org/wiki/Q2393193","display_name":"Running time","level":2,"score":0.2563000023365021},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2524999976158142}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3806645.3807813","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3806645.3807813","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2510.13668","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.13668","pdf_url":"https://arxiv.org/pdf/2510.13668","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:arXiv.org:2510.13668","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.13668","pdf_url":"https://arxiv.org/pdf/2510.13668","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.13668","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.13668","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3806645.3807813","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3806645.3807813","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th International Symposium on High-Performance Parallel and Distributed Computing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Model":[2],"(LLM)":[3],"inference":[4],"has":[5],"emerged":[6],"as":[7,32],"a":[8,60,117],"fundamental":[9],"paradigm,":[10],"however,":[11],"variations":[12],"in":[13,20,44,113],"output":[14],"length":[15,66,93],"cause":[16],"severe":[17],"workload":[18],"imbalance":[19],"the":[21],"decode":[22,52,61,114],"phase,":[23],"particularly":[24],"for":[25],"long-output":[26],"reasoning":[27],"tasks.":[28],"Existing":[29],"systems,":[30],"such":[31],"PD":[33],"disaggregation":[34],"architectures,":[35],"rely":[36],"on":[37],"static":[38],"prefill-to-decode":[39],"scheduling,":[40],"which":[41],"often":[42],"results":[43],"SLO":[45],"violations":[46],"and":[47,79,101,124,132],"OOM":[48],"failures":[49],"under":[50],"evolving":[51],"workloads.":[53,71],"In":[54],"this":[55],"paper,":[56],"we":[57],"propose":[58],"STAR,":[59],"rescheduling":[62,111],"system":[63],"powered":[64],"by":[65,99,107,130],"prediction":[67,82],"to":[68,89],"anticipate":[69],"future":[70],"Our":[72],"core":[73],"contributions":[74],"include:":[75],"(1)":[76],"A":[77,110],"lightweight":[78],"continuous":[80],"LLM-native":[81],"method":[83],"that":[84,121],"leverages":[85],"LLM":[86],"hidden":[87],"state":[88],"model":[90],"remaining":[91],"generation":[92],"with":[94,116],"high":[95],"precision":[96],"(reducing":[97],"MAE":[98],"49.42%)":[100],"low":[102],"overhead":[103],"(cutting":[104],"predictor":[105],"parameters":[106],"93.28%);":[108],"(2)":[109],"solution":[112],"phase":[115],"dynamic":[118],"balancing":[119],"mechanism":[120],"integrates":[122],"current":[123],"predicted":[125],"workloads,":[126],"reducing":[127],"P99":[128],"TPOT":[129],"75.1%":[131],"achieving":[133],"2.63":[134],"\u00d7":[135],"higher":[136],"goodput.":[137]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-17T00:00:00"}