{"id":"https://openalex.org/W7163550481","doi":"https://doi.org/10.48550/arxiv.2606.04415","title":"FlexNPU: Transparent NPU Virtualization for Dynamic LLM Prefill-Decode Co-location","display_name":"FlexNPU: Transparent NPU Virtualization for Dynamic LLM Prefill-Decode Co-location","publication_year":2026,"publication_date":"2026-06-03","ids":{"openalex":"https://openalex.org/W7163550481","doi":"https://doi.org/10.48550/arxiv.2606.04415"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.04415","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.04415","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.04415","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102628994","display_name":"Jiongjiong Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Jiongjiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137855126","display_name":"Jianfeng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jianfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137820549","display_name":"Zidong Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Zidong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137901983","display_name":"Yongqiao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yongqiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123004503","display_name":"Pengfei Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Pengfei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137901363","display_name":"Mingjie Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Mingjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137814665","display_name":"Hong Liu (50694)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Hong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102332577","display_name":"Yuanyi Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Yuanyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080825533","display_name":"Jiajia Chu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chu, Jiajia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111472589","display_name":"Yifeng Tang","orcid":"https://orcid.org/0009-0009-0921-4930"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Yifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137815551","display_name":"Hui Zang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zang, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137905439","display_name":"Xin Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101278305","display_name":"Qijie Qiu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Qijie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137881100","display_name":"Yuzhao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yuzhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137913292","display_name":"Chuanfei Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Chuanfei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137915855","display_name":"Lin Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137916192","display_name":"Zhuonan Lai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lai, Zhuonan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021811009","display_name":"Hongming Huang","orcid":"https://orcid.org/0000-0002-5531-474X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Hongming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130959099","display_name":"Jiawei Qiu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Jiawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137892006","display_name":"Gong Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Gong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137813899","display_name":"Zhong Ming","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Weipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137883925","display_name":"Weipeng Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ming, Zhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3659000098705292,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3659000098705292,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.09799999743700027,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.07859999686479568,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/virtualization","display_name":"Virtualization","score":0.6635000109672546},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.557699978351593},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.42170000076293945},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.3686000108718872},{"id":"https://openalex.org/keywords/permission","display_name":"Permission","score":0.33079999685287476},{"id":"https://openalex.org/keywords/decoupling","display_name":"Decoupling (probability)","score":0.3246000111103058},{"id":"https://openalex.org/keywords/petri-net","display_name":"Petri net","score":0.30970001220703125}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.824999988079071},{"id":"https://openalex.org/C513985346","wikidata":"https://www.wikidata.org/wiki/Q270471","display_name":"Virtualization","level":3,"score":0.6635000109672546},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.557699978351593},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.453900009393692},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.42170000076293945},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.3686000108718872},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.33219999074935913},{"id":"https://openalex.org/C2779089604","wikidata":"https://www.wikidata.org/wiki/Q7169333","display_name":"Permission","level":2,"score":0.33079999685287476},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.3246000111103058},{"id":"https://openalex.org/C38677869","wikidata":"https://www.wikidata.org/wiki/Q724168","display_name":"Petri net","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.30720001459121704},{"id":"https://openalex.org/C2984822820","wikidata":"https://www.wikidata.org/wiki/Q1123036","display_name":"Processor scheduling","level":3,"score":0.2964000105857849},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2915000021457672},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.28619998693466187},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C144240696","wikidata":"https://www.wikidata.org/wiki/Q367204","display_name":"Address space","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C2777145635","wikidata":"https://www.wikidata.org/wiki/Q515636","display_name":"FIFO (computing and electronics)","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C2778234956","wikidata":"https://www.wikidata.org/wiki/Q4683813","display_name":"Admission control","level":3,"score":0.2596000134944916},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.2533000111579895}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.04415","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.04415","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.04415","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.04415","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Modern":[0],"AI":[1,119],"serving":[2],"increasingly":[3],"relies":[4],"on":[5,99,166],"NPUs":[6,169],"for":[7,94,140,249],"conventional":[8],"inference":[9,186],"and":[10,32,67,82,102,136,155,170,188,212,251],"large":[11],"language":[12],"model":[13,117],"serving.":[14,142,254],"However,":[15],"current":[16],"NPU":[17,104,113,122,131,180,243],"deployments":[18],"commonly":[19],"expose":[20],"physical":[21,112],"devices":[22,114],"directly":[23],"to":[24,36,39,129,158],"applications,":[25],"which":[26,150],"limits":[27],"runtime":[28,125],"control":[29,133],"over":[30,206,229],"scheduling":[31,139,152],"makes":[33],"it":[34,172],"difficult":[35],"adapt":[37],"execution":[38],"phase-level":[40],"workload":[41],"behavior.":[42],"This":[43,124],"limitation":[44],"is":[45,55,61,245],"particularly":[46],"evident":[47],"in":[48,192],"LLM":[49,141,175,253],"serving,":[50],"where":[51],"the":[52,58],"prefill":[53,154],"phase":[54,60,75],"compute-intensive":[56],"while":[57,225],"decode":[59,156],"often":[62],"constrained":[63],"by":[64,210,228],"memory":[65],"bandwidth":[66],"KV-cache":[68],"accesses.":[69],"Static":[70],"prefill-decode":[71],"(PD)":[72],"disaggregation":[73,209],"reduces":[74],"interference,":[76],"but":[77],"can":[78],"introduce":[79],"resource":[80,161],"imbalance":[81],"unnecessary":[83],"data":[84],"movement.":[85],"We":[86,163],"present":[87],"FlexNPU,":[88],"a":[89,196,246],"transparent":[90,242],"user-space":[91],"virtualization":[92,244],"layer":[93],"Ascend":[95,168,198],"NPUs.":[96],"FlexNPU":[97,128,145,165,182,203,221],"interposes":[98],"AscendCL":[100],"APIs":[101],"routes":[103],"operations":[105],"through":[106],"per-device":[107],"daemons,":[108],"decoupling":[109],"unmodified":[110],"from":[111],"without":[115],"modifying":[116],"code,":[118],"frameworks,":[120],"or":[121],"drivers.":[123],"boundary":[126],"allows":[127],"virtualize":[130],"objects,":[132],"operator":[134],"dispatch,":[135],"support":[137],"phase-aware":[138],"In":[143],"particular,":[144],"enables":[146],"dynamic":[147],"PD":[148,208,219],"co-location,":[149,220],"adapts":[151],"between":[153],"according":[157],"their":[159],"complementary":[160],"characteristics.":[162],"implement":[164],"Huawei":[167],"evaluate":[171],"with":[173,178,217,234],"typical":[174],"workloads.":[176],"Compared":[177],"direct":[179],"passthrough,":[181],"introduces":[183],"no":[184],"measurable":[185],"overhead":[187],"slightly":[189],"improves":[190,204],"throughput":[191,205,224],"some":[193],"scenarios.":[194],"On":[195,214],"384-card":[197],"910C":[199],"deployment":[200],"of":[201],"DeepSeek-R1,":[202],"static":[207,218],"5.15%":[211],"26.33%.":[213],"Qwen2.5-7B,":[215],"compared":[216],"maintains":[222],"comparable":[223],"reducing":[226],"TTFT":[227],"92%":[230],"across":[231],"tested":[232],"workloads":[233],"nearly":[235],"unchanged":[236],"TPOT.":[237],"These":[238],"results":[239],"show":[240],"that":[241],"practical":[247],"substrate":[248],"efficient":[250],"responsive":[252]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-05T00:00:00"}