{"id":"https://openalex.org/W7161772187","doi":"https://doi.org/10.48550/arxiv.2605.19593","title":"Towards Multi-Model LLM Schedulers: Empirical Insights into Offloading and Preemption","display_name":"Towards Multi-Model LLM Schedulers: Empirical Insights into Offloading and Preemption","publication_year":2026,"publication_date":"2026-05-19","ids":{"openalex":"https://openalex.org/W7161772187","doi":"https://doi.org/10.48550/arxiv.2605.19593"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.19593","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19593","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.19593","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111394180","display_name":"Mert Yildiz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yildiz, Mert","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043668931","display_name":"Pietro Spadaccino","orcid":"https://orcid.org/0000-0002-5920-680X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Spadaccino, Pietro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136260651","display_name":"Alexey Rolich","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rolich, Alexey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136543438","display_name":"Francesca Cuomo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cuomo, Francesca","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5074428250","display_name":"Andrea Baiocchi","orcid":"https://orcid.org/0000-0002-9337-9421"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baiocchi, Andrea","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.45719999074935913,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.45719999074935913,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.15649999678134918,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.061500001698732376,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preemption","display_name":"Preemption","score":0.8647000193595886},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5810999870300293},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.508400022983551},{"id":"https://openalex.org/keywords/cache","display_name":"Cache","score":0.5027999877929688},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.4584999978542328},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4456000030040741},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.41359999775886536},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.4049000144004822}],"concepts":[{"id":"https://openalex.org/C206952183","wikidata":"https://www.wikidata.org/wiki/Q1193100","display_name":"Preemption","level":2,"score":0.8647000193595886},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8355000019073486},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5810999870300293},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.508400022983551},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.5027999877929688},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.49410000443458557},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4584999978542328},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4456000030040741},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.41359999775886536},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.4049000144004822},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3968999981880188},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.34880000352859497},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3384000062942505},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.29760000109672546},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.28130000829696655},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.2619999945163727},{"id":"https://openalex.org/C189783530","wikidata":"https://www.wikidata.org/wiki/Q352090","display_name":"CPU cache","level":3,"score":0.2578999996185303},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.25110000371932983}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.19593","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19593","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.19593","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19593","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Modern":[0],"deployments":[1],"of":[2,73,86,150,171,189,201,207],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"increasingly":[7],"require":[8],"serving":[9,204],"multiple":[10],"models":[11,107,141],"with":[12,105,213],"diverse":[13],"architectures,":[14],"sizes,":[15],"and":[16,31,42,89,99,134,142,153,160,185,191],"specialization":[17],"on":[18,82,164],"shared,":[19],"heterogeneous":[20],"hardware.":[21],"This":[22],"setting":[23],"introduces":[24],"new":[25],"challenges":[26],"for":[27,52,198],"resource":[28],"allocation,":[29],"dispatching,":[30],"scheduling,":[32],"particularly":[33],"under":[34,62],"GPU":[35,113],"memory":[36],"constraints":[37],"where":[38],"partial":[39],"CPU-GPU":[40,215],"offloading":[41,88,94,181],"preemption":[43,119,190],"become":[44],"necessary.":[45],"While":[46],"existing":[47],"systems":[48,205],"primarily":[49],"optimize":[50],"throughput":[51],"a":[53,169],"single":[54],"model,":[55],"comparatively":[56],"little":[57],"work":[58],"addresses":[59],"multi-model":[60,211],"scheduling":[61],"these":[63,165],"conditions.":[64],"In":[65],"this":[66,136],"paper,":[67],"we":[68,146,167],"present":[69],"an":[70],"empirical":[71],"study":[72],"how":[74],"different":[75],"LLMs":[76],"behave":[77],"across":[78,140],"hardware":[79,143],"platforms,":[80],"focusing":[81],"the":[83,148,186,199],"performance":[84],"implications":[85],"layer":[87],"preemption.":[90],"We":[91,115],"show":[92],"that":[93,118,135,174],"leads":[95],"to":[96,111],"strongly":[97],"non-linear":[98],"model-dependent":[100],"degradation":[101],"in":[102,156],"decode":[103],"throughput,":[104],"smaller":[106],"exhibiting":[108],"sharper":[109],"sensitivity":[110],"reduced":[112],"residency.":[114],"further":[116],"demonstrate":[117],"incurs":[120],"substantial":[121],"overhead,":[122],"largely":[123],"dominated":[124],"by":[125],"model":[126],"state":[127],"reload":[128],"rather":[129],"than":[130],"key-value":[131],"cache":[132],"transfer,":[133],"cost":[137,187],"varies":[138],"significantly":[139],"platforms.":[144],"Additionally,":[145],"highlight":[147],"role":[149],"sequence":[151],"length":[152],"interconnect":[154],"bandwidth":[155],"amplifying":[157],"data":[158,192],"movement":[159],"execution":[161],"inefficiencies.":[162],"Based":[163],"findings,":[166],"identify":[168],"set":[170],"key":[172],"features":[173],"future":[175],"schedulers":[176],"must":[177],"consider,":[178],"including":[179],"model-specific":[180],"sensitivity,":[182],"workload":[183],"characteristics,":[184],"structure":[188],"transfer.":[193],"These":[194],"insights":[195],"provide":[196],"guidance":[197],"design":[200],"next-generation":[202],"LLM":[203],"capable":[206],"efficiently":[208],"managing":[209],"heterogeneous,":[210],"workloads":[212],"hybrid":[214],"execution.":[216]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-21T00:00:00"}