{"id":"https://openalex.org/W4403576261","doi":"https://doi.org/10.48550/arxiv.2410.10759","title":"SplitLLM: Collaborative Inference of LLMs for Model Placement and Throughput Optimization","display_name":"SplitLLM: Collaborative Inference of LLMs for Model Placement and Throughput Optimization","alternate_titles":[],"publication_year":2024,"publication_date":"2024-10-14","ids":{"openalex":"https://openalex.org/W4403576261","doi":"https://doi.org/10.48550/arxiv.2410.10759"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.10759","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.10759","pdf_url":"https://arxiv.org/pdf/2410.10759","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4403576261"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.10759","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036108775","display_name":"Akrit Mudvari","orcid":"https://orcid.org/0009-0008-5607-4689","observed_orcids":["https://orcid.org/0009-0008-5607-4689"]},"institutions":[{"id":"https://openalex.org/I4210107353","display_name":"NEC (United States)","ror":"https://ror.org/01v791m31","country_code":"US","type":"company","lineage":["https://openalex.org/I118347220","https://openalex.org/I4210107353"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Mudvari, Akrit","raw_affiliation_strings":["Yale University NEC Laboratories America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University NEC Laboratories America","institution_ids":["https://openalex.org/I4210107353"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079671320","display_name":"Yuang Jiang","orcid":"https://orcid.org/0000-0001-8263-6085","observed_orcids":["https://orcid.org/0000-0001-8263-6085"]},"institutions":[{"id":"https://openalex.org/I4210107353","display_name":"NEC (United States)","ror":"https://ror.org/01v791m31","country_code":"US","type":"company","lineage":["https://openalex.org/I118347220","https://openalex.org/I4210107353"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jiang, Yuang","raw_affiliation_strings":["NEC Laboratories America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NEC Laboratories America","institution_ids":["https://openalex.org/I4210107353"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014892027","display_name":"Leandros Tassiulas","orcid":"https://orcid.org/0000-0003-0932-774X","observed_orcids":["https://orcid.org/0000-0003-0932-774X"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Tassiulas, Leandros","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":29,"reviewed_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.3984000086784363,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.3984000086784363,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.34299999475479126,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14098","display_name":"Energy Efficiency in Computing","score":0.05139999836683273,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8339999914169312},{"id":"https://openalex.org/keywords/collaborative-inference","display_name":"collaborative inference","score":0.9350000023841858},{"id":"https://openalex.org/keywords/model-placement","display_name":"model placement","score":0.7599999904632568},{"id":"https://openalex.org/keywords/inference-throughput","display_name":"inference throughput","score":0.597000002861023},{"id":"https://openalex.org/keywords/dynamic-programming","display_name":"dynamic programming","score":0.8180000185966492},{"id":"https://openalex.org/keywords/self-attention-complexity","display_name":"self-attention complexity","score":0.6549999713897705},{"id":"https://openalex.org/keywords/service-level-agreements","display_name":"service level agreements","score":0.8309999704360962},{"id":"https://openalex.org/keywords/resource-allocation","display_name":"resource allocation","score":0.4009999930858612},{"id":"https://openalex.org/keywords/communication-overhead","display_name":"communication overhead","score":0.4359999895095825}],"concepts":[{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.7699486017227173},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7422230243682861},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.502669095993042},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2321712076663971},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.07267284393310547},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.10759","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.10759","pdf_url":"https://arxiv.org/pdf/2410.10759","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.10759","doi":"https://doi.org/10.48550/arxiv.2410.10759","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.10759","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.10759","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.10759","pdf_url":"https://arxiv.org/pdf/2410.10759","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403576261.pdf","grobid_xml":"https://content.openalex.org/works/W4403576261.grobid-xml"},"referenced_works_count":50,"referenced_works":["https://openalex.org/W1995415305","https://openalex.org/W2108598243","https://openalex.org/W2119144962","https://openalex.org/W2300242332","https://openalex.org/W2468875367","https://openalex.org/W2495425901","https://openalex.org/W2528523909","https://openalex.org/W2786348542","https://openalex.org/W2798603777","https://openalex.org/W2808052525","https://openalex.org/W2896457183","https://openalex.org/W2911489562","https://openalex.org/W2912298604","https://openalex.org/W2918335825","https://openalex.org/W2939710050","https://openalex.org/W2945908221","https://openalex.org/W2955419283","https://openalex.org/W2963363373","https://openalex.org/W2964266063","https://openalex.org/W2975059944","https://openalex.org/W2982333655","https://openalex.org/W2990789643","https://openalex.org/W2994673210","https://openalex.org/W3009116850","https://openalex.org/W3015468748","https://openalex.org/W3033529678","https://openalex.org/W3045733172","https://openalex.org/W3091156754","https://openalex.org/W3092129462","https://openalex.org/W3092377875","https://openalex.org/W3103682594","https://openalex.org/W3107794213","https://openalex.org/W3122157069","https://openalex.org/W3190618479","https://openalex.org/W3193886995","https://openalex.org/W3196226093","https://openalex.org/W3208837070","https://openalex.org/W3214693004","https://openalex.org/W4210587557","https://openalex.org/W4221146017","https://openalex.org/W4236099117","https://openalex.org/W4286681622","https://openalex.org/W4296982580","https://openalex.org/W4300917616","https://openalex.org/W4312412889","https://openalex.org/W4318812474","https://openalex.org/W4366989525","https://openalex.org/W4383749512","https://openalex.org/W4386536290","https://openalex.org/W4401211704"],"related_works":["https://openalex.org/W7118047847","https://openalex.org/W7128638434","https://openalex.org/W4392489911","https://openalex.org/W4416159316","https://openalex.org/W4411403202","https://openalex.org/W7133235396","https://openalex.org/W4405173146","https://openalex.org/W4376167141","https://openalex.org/W4401211704","https://openalex.org/W4416905144"],"abstract_inverted_index":{"Large":[0],"language":[1,36],"models":[2],"(LLMs)":[3],"have":[4],"been":[5],"a":[6,15,119,124,153,210,214],"disruptive":[7],"innovation":[8],"in":[9,18,58,69,79,183,200,222],"recent":[10],"years,":[11],"and":[12,28,40,46,84,100,126,148,164],"they":[13],"play":[14],"crucial":[16],"role":[17],"our":[19],"daily":[20],"lives":[21],"due":[22],"to":[23,26,88,129,157,168,190,219],"their":[24],"ability":[25],"understand":[27],"generate":[29],"human-like":[30],"text.":[31],"Their":[32],"capabilities":[33],"include":[34],"natural":[35],"understanding,":[37],"information":[38],"retrieval":[39],"search,":[41],"translation,":[42],"chatbots,":[43],"virtual":[44],"assistance,":[45],"many":[47],"more.":[48],"However,":[49],"it":[50],"is":[51,98,107,237],"well":[52],"known":[53],"that":[54,186],"LLMs":[55],"are":[56,188,217],"massive":[57],"terms":[59,80],"of":[60,63,73,81,104,228,234],"the":[61,66,70,89,102,111,131,139,146,162,165,170,176,184,193,201,232,235],"number":[62],"parameters.":[64],"Additionally,":[65],"self-attention":[67],"mechanism":[68],"underlying":[71],"architecture":[72,122],"LLMs,":[74],"Transformers,":[75],"has":[76],"quadratic":[77],"complexity":[78],"both":[82,143],"computation":[83,147,160],"memory":[85],"with":[86,225],"respect":[87],"input":[90],"sequence":[91],"length.":[92],"For":[93],"these":[94],"reasons,":[95],"LLM":[96,105,229],"inference":[97,106,121,230],"resource-intensive,":[99],"thus,":[101],"throughput":[103,132,233],"limited,":[108],"especially":[109],"for":[110,196],"longer":[112],"sequences.":[113],"In":[114,134],"this":[115,135],"report,":[116],"we":[117,137,187,216],"design":[118],"collaborative":[120],"between":[123,161],"server":[125,163,171,202,236],"its":[127],"clients":[128],"alleviate":[130],"limit.":[133],"design,":[136],"consider":[138],"available":[140],"resources":[141],"on":[142],"sides,":[144],"i.e.,":[145],"communication":[149],"costs.":[150],"We":[151,181],"develop":[152],"dynamic":[154],"programming-based":[155],"algorithm":[156],"optimally":[158],"allocate":[159],"client":[166],"device":[167],"increase":[169],"throughput,":[172],"while":[173,204],"not":[174],"violating":[175],"service":[177],"level":[178],"agreement":[179],"(SLA).":[180],"show":[182],"experiments":[185],"able":[189,218],"efficiently":[191],"distribute":[192],"workload":[194],"allowing":[195],"roughly":[197],"1/3":[198],"reduction":[199],"workload,":[203],"achieving":[205],"19":[206],"percent":[207],"improvement":[208],"over":[209],"greedy":[211],"method.":[212],"As":[213],"result,":[215],"demonstrate":[220],"that,":[221],"an":[223],"environment":[224],"different":[226],"types":[227],"requests,":[231],"improved.":[238]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4403576261","counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":11}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2024-10-20T00:00:00"}
