{"id":"https://openalex.org/W4416123789","doi":"https://doi.org/10.48550/arxiv.2504.03702","title":"Hierarchical Prediction-based Management for LMaaS Systems","display_name":"Hierarchical Prediction-based Management for LMaaS Systems","publication_year":2025,"publication_date":"2025-03-25","ids":{"openalex":"https://openalex.org/W4416123789","doi":"https://doi.org/10.48550/arxiv.2504.03702"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2504.03702","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.03702","pdf_url":"https://arxiv.org/pdf/2504.03702","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2504.03702","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015050387","display_name":"Zhihan Jiang","orcid":"https://orcid.org/0000-0001-5270-9351","observed_orcids":["https://orcid.org/0000-0001-5270-9351"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jiang, Zhihan","raw_affiliation_strings":["The Chinese University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100721729","display_name":"Yujie Huang","orcid":"https://orcid.org/0000-0002-0197-3774","observed_orcids":["https://orcid.org/0000-0002-0197-3774"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Huang, Yujie","raw_affiliation_strings":["The Chinese University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075077822","display_name":"Guangba Yu","orcid":"https://orcid.org/0000-0001-6195-9088","observed_orcids":["https://orcid.org/0000-0001-6195-9088"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yu, Guangba","raw_affiliation_strings":["The Chinese University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116335706","display_name":"Junjie Huang","orcid":"https://orcid.org/0009-0001-2253-7541","observed_orcids":["https://orcid.org/0009-0001-2253-7541"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Huang, Junjie","raw_affiliation_strings":["The Chinese University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042468232","display_name":"Jiazhen Gu","orcid":"https://orcid.org/0000-0002-5831-9474","observed_orcids":["https://orcid.org/0000-0002-5831-9474"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Gu, Jiazhen","raw_affiliation_strings":["The Chinese University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5127959574","display_name":"Michael R. Lyu","orcid":"https://orcid.org/0000-0002-3666-5798","observed_orcids":["https://orcid.org/0000-0002-3666-5798"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Lyu, Michael R.","raw_affiliation_strings":["The Chinese University of Hong Kong Hong Kong SAR , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong Hong Kong SAR , China","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14098","display_name":"Energy Efficiency in Computing","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8949999809265137},{"id":"https://openalex.org/keywords/hierarchical-prediction","display_name":"hierarchical prediction","score":0.6700000166893005},{"id":"https://openalex.org/keywords/workload-prediction","display_name":"workload prediction","score":0.7129999995231628},{"id":"https://openalex.org/keywords/resource-allocation","display_name":"resource allocation","score":0.875},{"id":"https://openalex.org/keywords/request-routing","display_name":"request routing","score":0.9739999771118164},{"id":"https://openalex.org/keywords/tail-latency","display_name":"tail latency","score":0.7459999918937683},{"id":"https://openalex.org/keywords/service-level-objectives","display_name":"service level objectives","score":0.9089999794960022}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7965999841690063},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6301000118255615},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.550000011920929},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5471000075340271},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.5051000118255615},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4999000132083893},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.47429999709129333},{"id":"https://openalex.org/C138959212","wikidata":"https://www.wikidata.org/wiki/Q1806783","display_name":"Load balancing (electrical power)","level":3,"score":0.4359000027179718},{"id":"https://openalex.org/C2780609101","wikidata":"https://www.wikidata.org/wiki/Q17156588","display_name":"Resource management (computing)","level":2,"score":0.4251999855041504},{"id":"https://openalex.org/C29202148","wikidata":"https://www.wikidata.org/wiki/Q287260","display_name":"Resource allocation","level":2,"score":0.42320001125335693},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.41100001335144043},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.3901999890804291},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.3846000134944916},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3765999972820282},{"id":"https://openalex.org/C181889124","wikidata":"https://www.wikidata.org/wiki/Q380204","display_name":"Service level","level":2,"score":0.37290000915527344},{"id":"https://openalex.org/C2777480716","wikidata":"https://www.wikidata.org/wiki/Q23582796","display_name":"Resource consumption","level":2,"score":0.36880001425743103},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.3393999934196472},{"id":"https://openalex.org/C2780165032","wikidata":"https://www.wikidata.org/wiki/Q16869822","display_name":"Energy consumption","level":2,"score":0.31189998984336853},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.30720001459121704},{"id":"https://openalex.org/C16151460","wikidata":"https://www.wikidata.org/wiki/Q3480069","display_name":"Service level objective","level":5,"score":0.29989999532699585},{"id":"https://openalex.org/C2778160497","wikidata":"https://www.wikidata.org/wiki/Q869830","display_name":"Service-level agreement","level":3,"score":0.29499998688697815},{"id":"https://openalex.org/C2779370713","wikidata":"https://www.wikidata.org/wiki/Q357554","display_name":"Load management","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C51332947","wikidata":"https://www.wikidata.org/wiki/Q1172305","display_name":"Shared resource","level":2,"score":0.25940001010894775},{"id":"https://openalex.org/C2778274720","wikidata":"https://www.wikidata.org/wiki/Q7315806","display_name":"Resource Management System","level":3,"score":0.25360000133514404}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2504.03702","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.03702","pdf_url":"https://arxiv.org/pdf/2504.03702","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2504.03702","doi":"https://doi.org/10.48550/arxiv.2504.03702","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2504.03702","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2504.03702","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.03702","pdf_url":"https://arxiv.org/pdf/2504.03702","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322942","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416123789.pdf","grobid_xml":"https://content.openalex.org/works/W4416123789.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W2525246036","https://openalex.org/W2799054028","https://openalex.org/W2978017171","https://openalex.org/W3094473903","https://openalex.org/W3152956381","https://openalex.org/W4321472175","https://openalex.org/W4384345728","https://openalex.org/W4396913050","https://openalex.org/W4399795610","https://openalex.org/W4400104234","https://openalex.org/W4401202681","https://openalex.org/W4401306886","https://openalex.org/W4401943272","https://openalex.org/W4403883633","https://openalex.org/W4405396136","https://openalex.org/W4406472599","https://openalex.org/W4412875469","https://openalex.org/W4414281281"],"related_works":["https://openalex.org/W4404386015","https://openalex.org/W7162195266","https://openalex.org/W7213306050","https://openalex.org/W4399453677","https://openalex.org/W7083812211","https://openalex.org/W4416927406","https://openalex.org/W7206181811","https://openalex.org/W7163919246","https://openalex.org/W7163557897","https://openalex.org/W7167465440"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"revolutionized":[5],"numerous":[6],"domains,":[7],"driving":[8],"the":[9,98,106,130],"rise":[10],"of":[11,18,101,108,132],"Language-Model-as-a-Service":[12],"(LMaaS)":[13],"platforms":[14,22],"that":[15,163],"process":[16],"millions":[17],"queries":[19],"daily.":[20],"These":[21],"must":[23],"minimize":[24],"latency":[25,170],"and":[26,56,90,120,146],"meet":[27],"Service":[28],"Level":[29],"Objectives":[30],"(SLOs)":[31],"while":[32,178],"optimizing":[33],"resource":[34,99,125,174],"usage.":[35],"However,":[36],"conventional":[37],"cloud":[38],"service":[39,54,78],"management":[40,69],"techniques,":[41],"designed":[42],"for":[43,48,112],"traditional":[44],"workloads,":[45],"are":[46],"suboptimal":[47],"LMaaS":[49,68],"due":[50],"to":[51,81,96],"its":[52],"dynamic":[53],"workloads":[55],"variable":[57],"request":[58,93,140],"loads.":[59],"To":[60],"address":[61],"this,":[62],"we":[63],"propose":[64],"PreServe,":[65],"a":[66,77,87,91,109],"tailored":[67],"framework":[70],"centered":[71],"on":[72,158],"hierarchical":[73],"prediction.":[74],"PreServe":[75,123,138,164],"incorporates":[76],"workload":[79],"predictor":[80,95],"estimate":[82],"periodic":[83],"token":[84],"density":[85],"at":[86],"coarse":[88],"granularity":[89],"novel":[92],"load":[94,110,153],"assess":[97],"demand":[100],"individual":[102],"LLM":[103,114],"requests,":[104],"enabling":[105],"construction":[107],"anticipator":[111],"each":[113],"instance.":[115],"By":[116],"integrating":[117],"both":[118,144],"long-term":[119],"short-term":[121],"predictions,":[122],"adjusts":[124],"allocation":[126],"in":[127],"advance,":[128],"mitigating":[129],"risks":[131],"instance":[133,149],"under-":[134],"or":[135],"over-provisioning.":[136],"Besides,":[137],"optimizes":[139],"routing":[141],"by":[142,171,176],"considering":[143],"current":[145],"anticipated":[147],"future":[148],"loads,":[150],"ensuring":[151],"balanced":[152],"distribution":[154],"across":[155],"instances.":[156],"Evaluations":[157],"real-world":[159],"production":[160],"datasets":[161],"show":[162],"outperforms":[165],"state-of-the-art":[166],"methods,":[167],"reducing":[168],"tail":[169],"41.3%,":[172],"cutting":[173],"consumption":[175],"49.38%,":[177],"incurring":[179],"only":[180],"0.23%":[181],"additional":[182],"overhead.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}