{"id":"https://openalex.org/W7160851357","doi":"https://doi.org/10.48550/arxiv.2605.06914","title":"Regulating Branch Parallelism in LLM Serving","display_name":"Regulating Branch Parallelism in LLM Serving","publication_year":2026,"publication_date":"2026-05-07","ids":{"openalex":"https://openalex.org/W7160851357","doi":"https://doi.org/10.48550/arxiv.2605.06914"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.06914","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06914","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.06914","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043630016","display_name":"Swapnil Gandhi","orcid":"https://orcid.org/0000-0003-3689-9591"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gandhi, Swapnil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107132155","display_name":"Siva Hari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hari, Siva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135907093","display_name":"William J. Dally","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dally, William J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135861016","display_name":"Christos Kozyrakis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kozyrakis, Christos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.30630001425743103,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.30630001425743103,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.19189999997615814,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.14910000562667847,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/goodput","display_name":"Goodput","score":0.6955999732017517},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6410999894142151},{"id":"https://openalex.org/keywords/context-switch","display_name":"Context switch","score":0.6055999994277954},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.5565000176429749},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.47600001096725464},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.41999998688697815},{"id":"https://openalex.org/keywords/prefix","display_name":"Prefix","score":0.39410001039505005},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.3440000116825104}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7706000208854675},{"id":"https://openalex.org/C94022561","wikidata":"https://www.wikidata.org/wiki/Q1172393","display_name":"Goodput","level":4,"score":0.6955999732017517},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6410999894142151},{"id":"https://openalex.org/C53833338","wikidata":"https://www.wikidata.org/wiki/Q1061424","display_name":"Context switch","level":2,"score":0.6055999994277954},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.5619999766349792},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.5565000176429749},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.5400999784469604},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.47600001096725464},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.41999998688697815},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.39410001039505005},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.38350000977516174},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.3440000116825104},{"id":"https://openalex.org/C75949130","wikidata":"https://www.wikidata.org/wiki/Q848010","display_name":"Database transaction","level":2,"score":0.3395000100135803},{"id":"https://openalex.org/C16118543","wikidata":"https://www.wikidata.org/wiki/Q275372","display_name":"Externality","level":2,"score":0.33799999952316284},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3208000063896179},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.3000999987125397},{"id":"https://openalex.org/C193702766","wikidata":"https://www.wikidata.org/wiki/Q1414548","display_name":"Concurrency","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2754000127315521},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.2700999975204468},{"id":"https://openalex.org/C162932704","wikidata":"https://www.wikidata.org/wiki/Q1058791","display_name":"Synchronizing","level":3,"score":0.26930001378059387},{"id":"https://openalex.org/C2779084733","wikidata":"https://www.wikidata.org/wiki/Q17097343","display_name":"Processor sharing","level":3,"score":0.2669000029563904},{"id":"https://openalex.org/C203479927","wikidata":"https://www.wikidata.org/wiki/Q5165939","display_name":"Controller (irrigation)","level":2,"score":0.26600000262260437},{"id":"https://openalex.org/C138101251","wikidata":"https://www.wikidata.org/wiki/Q213092","display_name":"Thread (computing)","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C180591934","wikidata":"https://www.wikidata.org/wiki/Q1253369","display_name":"Downtime","level":2,"score":0.25519999861717224},{"id":"https://openalex.org/C2780233049","wikidata":"https://www.wikidata.org/wiki/Q13134154","display_name":"Trichotomy (philosophy)","level":2,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.06914","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06914","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.06914","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06914","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"methods":[1],"expose":[2],"intra-request":[3],"parallelism":[4],"in":[5,41,55],"LLM":[6],"outputs,":[7],"allowing":[8],"independent":[9],"branches":[10,19,54,68,106,135],"to":[11],"decode":[12,36],"concurrently.":[13],"Existing":[14],"serving":[15],"systems":[16],"execute":[17],"these":[18],"eagerly":[20],"or":[21,143],"under":[22],"fixed":[23,46],"caps.":[24],"We":[25,59,96],"show":[26,73],"that":[27,51,74,103],"both":[28],"are":[29],"brittle:":[30],"eager":[31],"admission":[32,101],"inflates":[33],"the":[34,49,56,61,69,75,113,119,137],"shared":[35],"step,":[37],"degrading":[38],"co-batched":[39],"requests":[40],"serial":[42],"stages,":[43],"while":[44,164],"conservative":[45],"caps":[47],"forgo":[48],"throughput":[50],"motivated":[52],"exposing":[53],"first":[57],"place.":[58],"call":[60],"excess":[62],"step":[63],"latency":[64],"caused":[65],"by":[66,155,160],"admitted":[67,110],"branch":[70,115],"externality":[71,116],"and":[72,84,159],"safe":[76],"width":[77,145],"depends":[78],"on":[79],"batch":[80],"composition,":[81],"context":[82],"lengths,":[83],"accumulated":[85],"slack,":[86],"all":[87],"of":[88],"which":[89],"change":[90],"continuously":[91],"over":[92,157,162,166],"a":[93,99],"workload":[94],"trace.":[95],"introduce":[97],"TAPER,":[98],"per-step":[100],"controller":[102],"treats":[104],"extra":[105],"as":[107],"opportunistic":[108],"work,":[109],"only":[111],"when":[112],"predicted":[114],"fits":[117],"within":[118],"batch's":[120],"current":[121],"slack":[122],"budget.":[123],"Per-step":[124],"regulation":[125],"is":[126],"practical":[127],"because":[128],"branch-level":[129],"scheduling":[130],"decouples":[131],"compute":[132],"from":[133],"memory:":[134],"share":[136],"request's":[138],"prefix":[139],"KV,":[140],"so":[141],"expanding":[142],"contracting":[144],"requires":[146],"no":[147],"memory":[148],"reclamation.":[149],"On":[150],"Qwen3-32B,":[151],"TAPER":[152],"improves":[153],"goodput":[154],"$1.77\\times$":[156],"IRP-Off":[158],"$1.48\\times$":[161],"IRP-Eager,":[163],"maintaining":[165],"$95\\%$":[167],"SLO":[168],"attainment.":[169]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-12T00:00:00"}