{"id":"https://openalex.org/W7171984167","doi":"https://doi.org/10.1109/main71116.2026.11622343","title":"Towards Multi-Model LLM Schedulers: Empirical Insights Into Offloading and Preemption","display_name":"Towards Multi-Model LLM Schedulers: Empirical Insights Into Offloading and Preemption","alternate_titles":[],"publication_year":2026,"publication_date":"2026-07-01","ids":{"openalex":"https://openalex.org/W7171984167","doi":"https://doi.org/10.1109/main71116.2026.11622343"},"language":"en","primary_location":{"id":"doi:10.1109/main71116.2026.11622343","doi":"https://doi.org/10.1109/main71116.2026.11622343","is_oa":false,"landing_page_url":"https://doi.org/10.1109/main71116.2026.11622343","pdf_url":null,"source":{"id":"https://openalex.org/S7407120230","display_name":"Mediterranean Artificial Intelligence and Networking Conference (MAIN)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2026 Mediterranean Artificial Intelligence and Networking Conference (MAIN)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W7171984167"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2605.19593","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111394180","display_name":"Mert Yildiz","orcid":"https://orcid.org/0009-0005-3624-0190","observed_orcids":["https://orcid.org/0009-0005-3624-0190"]},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Mert Yildiz","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043668931","display_name":"Pietro Spadaccino","orcid":"https://orcid.org/0000-0002-5920-680X","observed_orcids":["https://orcid.org/0000-0002-5920-680X"]},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Pietro Spadaccino","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031873534","display_name":"Alexey Rolich","orcid":"https://orcid.org/0000-0001-8699-1084","observed_orcids":["https://orcid.org/0000-0001-8699-1084"]},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Alexey Rolich","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022341678","display_name":"Francesca Cuomo","orcid":"https://orcid.org/0000-0002-9122-7993","observed_orcids":["https://orcid.org/0000-0002-9122-7993"]},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Francesca Cuomo","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074428250","display_name":"Andrea Baiocchi","orcid":"https://orcid.org/0000-0002-9337-9421","observed_orcids":["https://orcid.org/0000-0002-9337-9421"]},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Andrea Baiocchi","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I861853513"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9639000296592712,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9639000296592712,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.012000000104308128,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.004999999888241291,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preemption","display_name":"preemption","score":0.8420000076293945},{"id":"https://openalex.org/keywords/gpu-memory-constraints","display_name":"GPU memory constraints","score":0.7919999957084656},{"id":"https://openalex.org/keywords/interconnect-bandwidth","display_name":"interconnect bandwidth","score":0.8090000152587891},{"id":"https://openalex.org/keywords/sequence-length","display_name":"sequence length","score":0.753000020980835},{"id":"https://openalex.org/keywords/heterogeneous-hardware","display_name":"heterogeneous hardware","score":0.8240000009536743},{"id":"https://openalex.org/keywords/resource-allocation","display_name":"resource allocation","score":0.7440000176429749}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5482000112533569},{"id":"https://openalex.org/C206952183","wikidata":"https://www.wikidata.org/wiki/Q1193100","display_name":"Preemption","level":2,"score":0.4724999964237213},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.29159998893737793},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2816999852657318},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.27239999175071716},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25699999928474426},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.24950000643730164},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.24199999868869781},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.23070000112056732}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/main71116.2026.11622343","doi":"https://doi.org/10.1109/main71116.2026.11622343","is_oa":false,"landing_page_url":"https://doi.org/10.1109/main71116.2026.11622343","pdf_url":null,"source":{"id":"https://openalex.org/S7407120230","display_name":"Mediterranean Artificial Intelligence and Networking Conference (MAIN)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2026 Mediterranean Artificial Intelligence and Networking Conference (MAIN)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2605.19593","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.19593","pdf_url":"https://arxiv.org/pdf/2605.19593","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:iris.uniroma1.it:11573/1774302","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/11573/1774302","pdf_url":null,"source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"doi:10.48550/arxiv.2605.19593","doi":"https://doi.org/10.48550/arxiv.2605.19593","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19593","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2605.19593","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2605.19593","pdf_url":"https://arxiv.org/pdf/2605.19593","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4875262911","display_name":null,"funder_award_id":"CUP B53C22004050001","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G679135132","display_name":null,"funder_award_id":"B53C22004050001","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7205771190","display_name":null,"funder_award_id":"PE00000001","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7171984167.pdf","grobid_xml":"https://content.openalex.org/works/W7171984167.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W4324297016","https://openalex.org/W4376167141","https://openalex.org/W4386395487","https://openalex.org/W4387321091","https://openalex.org/W4391045969","https://openalex.org/W4391272922","https://openalex.org/W4391800686","https://openalex.org/W4392488876","https://openalex.org/W4392489911","https://openalex.org/W4393941946","https://openalex.org/W4394998727","https://openalex.org/W4399453677","https://openalex.org/W4400222625","https://openalex.org/W4401211704","https://openalex.org/W4404351261","https://openalex.org/W4404401018","https://openalex.org/W4404995266","https://openalex.org/W4409248688","https://openalex.org/W4411486062","https://openalex.org/W4413145732","https://openalex.org/W4414736068","https://openalex.org/W4415799217","https://openalex.org/W4415937934","https://openalex.org/W4416011790","https://openalex.org/W7084057557","https://openalex.org/W7133235396","https://openalex.org/W7133238857"],"related_works":["https://openalex.org/W7117769016","https://openalex.org/W7127603034","https://openalex.org/W4417291540","https://openalex.org/W4404307937","https://openalex.org/W4415136390","https://openalex.org/W7162194745","https://openalex.org/W7082320824","https://openalex.org/W7104010441","https://openalex.org/W7117682392","https://openalex.org/W7221815260"],"abstract_inverted_index":{"Modern":[0],"deployments":[1],"of":[2,73,86,150,171,189,201,207],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"increasingly":[7],"require":[8],"serving":[9,204],"multiple":[10],"models":[11,107,141],"with":[12,105,213],"diverse":[13],"architectures,":[14],"sizes,":[15],"and":[16,31,42,89,99,134,142,153,160,185,191],"specialization":[17],"on":[18,82,164],"shared,":[19],"heterogeneous":[20],"hardware.":[21],"This":[22],"setting":[23],"introduces":[24],"new":[25],"challenges":[26],"for":[27,52,198],"resource":[28],"allocation,":[29],"dispatching,":[30],"scheduling,":[32],"particularly":[33],"under":[34,62],"GPU":[35,113],"memory":[36],"constraints":[37],"where":[38],"partial":[39],"CPU-GPU":[40,215],"offloading":[41,88,94,181],"preemption":[43,119,190],"become":[44],"necessary.":[45],"While":[46],"existing":[47],"systems":[48,205],"primarily":[49],"optimize":[50],"throughput":[51],"a":[53,169],"single":[54],"model,":[55],"comparatively":[56],"little":[57],"work":[58],"addresses":[59],"multi-model":[60,211],"scheduling":[61],"these":[63,165],"conditions.":[64],"In":[65],"this":[66,136],"paper,":[67],"we":[68,146,167],"present":[69],"an":[70],"empirical":[71],"study":[72],"how":[74],"different":[75],"LLMs":[76],"behave":[77],"across":[78,140],"hardware":[79,143],"platforms,":[80],"focusing":[81],"the":[83,148,186,199],"performance":[84],"implications":[85],"layer":[87],"preemption.":[90],"We":[91,115],"show":[92],"that":[93,118,135,174],"leads":[95],"to":[96,111],"strongly":[97],"non-linear":[98],"model-dependent":[100],"degradation":[101],"in":[102,156],"decode":[103],"throughput,":[104],"smaller":[106],"exhibiting":[108],"sharper":[109],"sensitivity":[110],"reduced":[112],"residency.":[114],"further":[116],"demonstrate":[117],"incurs":[120],"substantial":[121],"overhead,":[122],"largely":[123],"dominated":[124],"by":[125],"model":[126],"state":[127],"reload":[128],"rather":[129],"than":[130],"key-value":[131],"cache":[132],"transfer,":[133],"cost":[137,187],"varies":[138],"significantly":[139],"platforms.":[144],"Additionally,":[145],"highlight":[147],"role":[149],"sequence":[151],"length":[152],"interconnect":[154],"bandwidth":[155],"amplifying":[157],"data":[158,192],"movement":[159],"execution":[161],"inefficiencies.":[162],"Based":[163],"findings,":[166],"identify":[168],"set":[170],"key":[172],"features":[173],"future":[175],"schedulers":[176],"must":[177],"consider,":[178],"including":[179],"model-specific":[180],"sensitivity,":[182],"workload":[183],"characteristics,":[184],"structure":[188],"transfer.":[193],"These":[194],"insights":[195],"provide":[196],"guidance":[197],"design":[200],"next-generation":[202],"LLM":[203],"capable":[206],"efficiently":[208],"managing":[209],"heterogeneous,":[210],"workloads":[212],"hybrid":[214],"execution.":[216]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7171984167","counts_by_year":[],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2026-08-01T00:00:00"}
