{"id":"https://openalex.org/W4379538751","doi":"https://doi.org/10.48550/arxiv.2306.02003","title":"On Optimal Caching and Model Multiplexing for Large Model Inference","display_name":"On Optimal Caching and Model Multiplexing for Large Model Inference","publication_year":2023,"publication_date":"2023-06-03","ids":{"openalex":"https://openalex.org/W4379538751","doi":"https://doi.org/10.48550/arxiv.2306.02003"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2306.02003","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.02003","pdf_url":"https://arxiv.org/pdf/2306.02003","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2306.02003","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031613608","display_name":"Banghua Zhu","orcid":"https://orcid.org/0000-0002-7320-3533","observed_orcids":["https://orcid.org/0000-0002-7320-3533"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Banghua","raw_affiliation_strings":["Department of Electrical Engineering and Computer Sciences , UC Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Sciences , UC Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083402013","display_name":"Ying Sheng","orcid":"https://orcid.org/0000-0002-1883-2126","observed_orcids":["https://orcid.org/0000-0002-1883-2126"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sheng, Ying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061339425","display_name":"Lianmin Zheng","orcid":"https://orcid.org/0000-0001-5812-731X","observed_orcids":["https://orcid.org/0000-0001-5812-731X"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zheng, Lianmin","raw_affiliation_strings":["Department of Electrical Engineering and Computer Sciences , UC Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Sciences , UC Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026961968","display_name":"Clark Barrett","orcid":"https://orcid.org/0000-0002-9522-3084","observed_orcids":["https://orcid.org/0000-0002-9522-3084"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barrett, Clark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049812527","display_name":"Michael I. Jordan","orcid":"https://orcid.org/0000-0001-8935-817X","observed_orcids":["https://orcid.org/0000-0001-8935-817X"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jordan, Michael I.","raw_affiliation_strings":["Department of Electrical Engineering and Computer Sciences , UC Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Sciences , UC Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034192173","display_name":"Jiantao Jiao","orcid":"https://orcid.org/0000-0003-3766-8031","observed_orcids":["https://orcid.org/0000-0003-3766-8031"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiao, Jiantao","raw_affiliation_strings":["Department of Electrical Engineering and Computer Sciences , UC Berkeley","Department of Statistics , UC Berkeley \u22c4 Computer Science Department , Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Sciences , UC Berkeley","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"Department of Statistics , UC Berkeley \u22c4 Computer Science Department , Stanford University","institution_ids":["https://openalex.org/I95457486","https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.23340000212192535,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.23340000212192535,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.13760000467300415,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.10440000146627426,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8519999980926514},{"id":"https://openalex.org/keywords/large-foundation-models","display_name":"large foundation models","score":0.6589999794960022},{"id":"https://openalex.org/keywords/inference-cost-reduction","display_name":"inference cost reduction","score":0.6859999895095825},{"id":"https://openalex.org/keywords/query-caching","display_name":"query caching","score":0.8709999918937683},{"id":"https://openalex.org/keywords/ensemble-model-selection","display_name":"ensemble model selection","score":0.8040000200271606},{"id":"https://openalex.org/keywords/flops-reduction","display_name":"FLOPs reduction","score":0.7609999775886536},{"id":"https://openalex.org/keywords/inference-latency","display_name":"inference latency","score":0.8500000238418579}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8087958097457886},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.730741560459137},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.6652950048446655},{"id":"https://openalex.org/C70970002","wikidata":"https://www.wikidata.org/wiki/Q189434","display_name":"Multiplexer","level":3,"score":0.5729026794433594},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5476868748664856},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.504712700843811},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.48425623774528503},{"id":"https://openalex.org/C19275194","wikidata":"https://www.wikidata.org/wiki/Q222903","display_name":"Multiplexing","level":2,"score":0.4581124186515808},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.45370763540267944},{"id":"https://openalex.org/C196921405","wikidata":"https://www.wikidata.org/wiki/Q786431","display_name":"Online algorithm","level":2,"score":0.4149366021156311},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2663496136665344},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2116771936416626},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.19375848770141602},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.17056643962860107},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2306.02003","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.02003","pdf_url":"https://arxiv.org/pdf/2306.02003","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2306.02003","doi":"https://doi.org/10.48550/arxiv.2306.02003","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2306.02003","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2306.02003","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.02003","pdf_url":"https://arxiv.org/pdf/2306.02003","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth","score":0.5699999928474426}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4379538751.pdf","grobid_xml":"https://content.openalex.org/works/W4379538751.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4323268213","https://openalex.org/W4242128654","https://openalex.org/W2039899645","https://openalex.org/W2371342700","https://openalex.org/W2327110311","https://openalex.org/W2363059241","https://openalex.org/W3215187042","https://openalex.org/W4233510838","https://openalex.org/W266342242","https://openalex.org/W2100439961"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"and":[4,20,58,92,124,136,160,185],"other":[5],"large":[6],"foundation":[7],"models":[8,30,69],"have":[9],"achieved":[10],"noteworthy":[11],"success,":[12],"but":[13],"their":[14],"size":[15],"exacerbates":[16],"existing":[17],"resource":[18,36],"consumption":[19],"latency":[21,190,196],"challenges.":[22],"In":[23,40],"particular,":[24],"the":[25,34,86,131,143,151,154,157,176,179,192],"large-scale":[26],"deployment":[27],"of":[28,68,133],"these":[29,49],"is":[31,163,183,197],"hindered":[32],"by":[33],"significant":[35],"requirements":[37],"during":[38],"inference.":[39],"this":[41],"paper,":[42],"we":[43,74,117],"study":[44],"two":[45],"approaches":[46,83],"for":[47,70,79,181,194],"mitigating":[48],"challenges:":[50],"employing":[51],"a":[52,60,98,114,170,186],"cache":[53],"to":[54,63,84,147],"store":[55],"previous":[56],"queries":[57],"learning":[59],"model":[61,115,137],"multiplexer":[62],"choose":[64],"from":[65],"an":[66,76],"ensemble":[67],"query":[71],"processing.":[72],"Theoretically,":[73],"provide":[75],"optimal":[77,119],"algorithm":[78],"jointly":[80],"optimizing":[81],"both":[82,90,122],"reduce":[85],"inference":[87],"cost":[88,159,162],"in":[89,121,173,189],"offline":[91,123],"online":[93,125],"tabular":[94],"settings.":[95,126],"By":[96],"combining":[97],"caching":[99,135],"algorithm,":[100],"namely":[101],"Greedy":[102],"Dual":[103],"Size":[104],"with":[105,113,145],"Frequency":[106],"(GDSF)":[107],"or":[108],"Least":[109],"Expected":[110],"Cost":[111],"(LEC),":[112],"multiplexer,":[116],"achieve":[118],"rates":[120],"Empirically,":[127],"simulations":[128],"show":[129,169],"that":[130],"combination":[132],"our":[134],"multiplexing":[138],"algorithms":[139],"greatly":[140],"improves":[141],"over":[142,150,175],"baselines,":[144],"up":[146],"$50\\times$":[148],"improvement":[149,172,188],"baseline":[152,177],"when":[153,178,191],"ratio":[155,180,193],"between":[156],"maximum":[158],"minimum":[161],"$100$.":[164],"Experiments":[165],"on":[166],"real":[167],"datasets":[168],"$4.3\\times$":[171],"FLOPs":[174,182],"$10$,":[184],"$1.8\\times$":[187],"average":[195],"$1.85$.":[198]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2025-10-10T00:00:00"}