{"id":"https://openalex.org/W4376167141","doi":"https://doi.org/10.48550/arxiv.2305.05920","title":"Fast Distributed Inference Serving for Large Language Models","display_name":"Fast Distributed Inference Serving for Large Language Models","publication_year":2023,"publication_date":"2023-05-10","ids":{"openalex":"https://openalex.org/W4376167141","doi":"https://doi.org/10.48550/arxiv.2305.05920"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2305.05920","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.05920","pdf_url":"https://arxiv.org/pdf/2305.05920","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.05920","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100660218","display_name":"Bingyang Wu","orcid":"https://orcid.org/0000-0001-7221-8007","observed_orcids":["https://orcid.org/0000-0001-7221-8007"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wu, Bingyang","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000644890","display_name":"Yinmin Zhong","orcid":"https://orcid.org/0000-0002-2504-7652","observed_orcids":["https://orcid.org/0000-0002-2504-7652"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhong, Yinmin","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100603105","display_name":"Zili Zhang","orcid":"https://orcid.org/0000-0002-8721-9333","observed_orcids":["https://orcid.org/0000-0002-8721-9333"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Zili","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Shengyu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Shengyu","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Fangyue","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Fangyue","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sun, Yuanhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sun, Yuanhang","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Huang, Gang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huang, Gang","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Xuanzhe","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Xuanzhe","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":null,"display_name":"Jin, Xin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jin, Xin","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I20231570"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":24,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.5713000297546387,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.5713000297546387,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.11469999700784683,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.11180000007152557,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8399999737739563},{"id":"https://openalex.org/keywords/distributed-inference","display_name":"distributed inference","score":0.8840000033378601},{"id":"https://openalex.org/keywords/preemptive-scheduling","display_name":"preemptive scheduling","score":0.8149999976158142},{"id":"https://openalex.org/keywords/head-of-line-blocking","display_name":"head-of-line blocking","score":0.9190000295639038},{"id":"https://openalex.org/keywords/latency-reduction","display_name":"latency reduction","score":0.6359999775886536},{"id":"https://openalex.org/keywords/gpu-memory-management","display_name":"GPU memory management","score":0.972000002861023}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7927370071411133},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6811758875846863},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.593421995639801},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5791442394256592},{"id":"https://openalex.org/C206952183","wikidata":"https://www.wikidata.org/wiki/Q1193100","display_name":"Preemption","level":2,"score":0.5408798456192017},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3617740273475647},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.35016506910324097},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2820432186126709},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.27779680490493774},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.22021469473838806},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0913781225681305},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.08936142921447754}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2305.05920","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.05920","pdf_url":"https://arxiv.org/pdf/2305.05920","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2305.05920","doi":"https://doi.org/10.48550/arxiv.2305.05920","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.05920","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.05920","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.05920","pdf_url":"https://arxiv.org/pdf/2305.05920","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4376167141.pdf","grobid_xml":"https://content.openalex.org/works/W4376167141.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4411403202","https://openalex.org/W4392489911","https://openalex.org/W4415797367","https://openalex.org/W4399453677","https://openalex.org/W4403785764","https://openalex.org/W7084052333","https://openalex.org/W4405099035","https://openalex.org/W4394839034","https://openalex.org/W4416034208","https://openalex.org/W4401306494"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"power":[4],"a":[5,48,81,155],"new":[6,91],"generation":[7],"of":[8,18,60,69,94,158],"interactive":[9,16],"AI":[10],"applications":[11,20],"exemplified":[12],"by":[13,175],"ChatGPT.":[14],"The":[15,116],"nature":[17],"these":[19],"demands":[21],"low":[22],"latency":[23,79,187],"for":[24,34,53,110,150],"LLM":[25,28,61,95,151],"inference.":[26,152],"Existing":[27],"serving":[29,51],"systems":[30],"use":[31],"run-to-completion":[32],"processing":[33],"inference":[35,50,62],"jobs,":[36],"which":[37],"suffers":[38],"from":[39],"head-of-line":[40],"blocking":[41],"and":[42,140,147,160,179,185],"long":[43],"latency.":[44],"We":[45,129,153],"present":[46],"FastServe,":[47],"distributed":[49],"system":[52,156],"LLMs.":[54],"FastServe":[55,73,159,171],"exploits":[56],"the":[57,67,90,97,100,121,167,173,182],"autoregressive":[58],"pattern":[59],"to":[63,77,104,114,126,166,177],"enable":[64],"preemption":[65],"at":[66],"granularity":[68],"each":[70,111],"output":[71],"token.":[72],"uses":[74],"preemptive":[75],"scheduling":[76],"minimize":[78],"with":[80],"novel":[82],"skip-join":[83],"Multi-Level":[84],"Feedback":[85],"Queue":[86],"scheduler.":[87],"Based":[88],"on":[89],"semi-information-agnostic":[92],"setting":[93],"inference,":[96],"scheduler":[98],"leverages":[99],"input":[101],"length":[102],"information":[103],"assign":[105],"an":[106,131],"appropriate":[107],"initial":[108],"queue":[109,123],"arrival":[112],"job":[113],"join.":[115],"higher":[117],"priority":[118],"queues":[119],"than":[120],"joined":[122],"are":[124],"skipped":[125],"reduce":[127],"demotions.":[128],"design":[130],"efficient":[132],"GPU":[133,145],"memory":[134,146,149],"management":[135],"mechanism":[136],"that":[137,164],"proactively":[138],"offloads":[139],"uploads":[141],"intermediate":[142],"state":[143],"between":[144],"host":[148],"build":[154],"prototype":[157],"experimental":[161],"results":[162],"show":[163],"compared":[165],"state-of-the-art":[168],"solution":[169],"vLLM,":[170],"improves":[172],"throughput":[174],"up":[176],"31.4x":[178],"17.9x":[180],"under":[181],"same":[183],"average":[184],"tail":[186],"requirements,":[188],"respectively.":[189]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2023-05-12T00:00:00"}