{"id":"https://openalex.org/W7226008802","doi":"https://doi.org/10.48550/arxiv.2611.14848","title":"Tideway: Phase-Aware Tiered Memory Management for MoE Inference on Consumer Devices","display_name":"Tideway: Phase-Aware Tiered Memory Management for MoE Inference on Consumer Devices","alternate_titles":[],"publication_year":2026,"publication_date":"2026-11-10","ids":{"openalex":"https://openalex.org/W7226008802","doi":"https://doi.org/10.48550/arxiv.2611.14848"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2611.14848","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.14848","pdf_url":"https://arxiv.org/pdf/2611.14848","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7226008802"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2611.14848","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5161925772","display_name":"Yifei Wang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yifei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073936068","display_name":"Tsun-Yu Yang","orcid":"https://orcid.org/0009-0002-8992-480X","observed_orcids":["https://orcid.org/0009-0002-8992-480X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Tsun-yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043067273","display_name":"Yechen Xu","orcid":"https://orcid.org/0009-0009-7390-1153","observed_orcids":["https://orcid.org/0009-0009-7390-1153"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Yechen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000892367","display_name":"Hancheng Ye","orcid":"https://orcid.org/0000-0002-6272-2792","observed_orcids":["https://orcid.org/0000-0002-6272-2792"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Hancheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5161890669","display_name":"Qinsi Wang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Qinsi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144354055","display_name":"Baijun Zhou","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Baijun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5161819920","display_name":"Tingjun Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Tingjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5161810525","display_name":"Hai \"Helen\" Li","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hai \"Helen\"","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007943946","display_name":"Danyang Zhuo","orcid":"https://orcid.org/0000-0002-0611-3941","observed_orcids":["https://orcid.org/0000-0002-0611-3941"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhuo, Danyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5161957663","display_name":"Yiran Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yiran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9843999743461609,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9843999743461609,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.003100000089034438,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0026000000070780516,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.9359999895095825},{"id":"https://openalex.org/keywords/memory-hierarchy","display_name":"memory hierarchy","score":0.7250000238418579},{"id":"https://openalex.org/keywords/cuda-graphs","display_name":"CUDA Graphs","score":0.7300000190734863},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.7519999742507935},{"id":"https://openalex.org/keywords/consumer-devices","display_name":"consumer devices","score":0.7609999775886536},{"id":"https://openalex.org/keywords/memory-constrained-deployment","display_name":"memory-constrained deployment","score":0.4819999933242798}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.84170001745224},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.666100025177002},{"id":"https://openalex.org/C126831891","wikidata":"https://www.wikidata.org/wiki/Q221673","display_name":"Host (biology)","level":2,"score":0.5508999824523926},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.521399974822998},{"id":"https://openalex.org/C28362024","wikidata":"https://www.wikidata.org/wiki/Q2067413","display_name":"Thrashing","level":2,"score":0.5070000290870667},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.4975000023841858},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3953999876976013},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.35589998960494995},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3431999981403351},{"id":"https://openalex.org/C197115733","wikidata":"https://www.wikidata.org/wiki/Q1003136","display_name":"Forcing (mathematics)","level":2,"score":0.3427000045776367},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.3391000032424927},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.33820000290870667},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.3325999975204468},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.32499998807907104},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.30970001220703125},{"id":"https://openalex.org/C46743427","wikidata":"https://www.wikidata.org/wiki/Q1341685","display_name":"Inference engine","level":3,"score":0.30300000309944153},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.29910001158714294},{"id":"https://openalex.org/C50954386","wikidata":"https://www.wikidata.org/wiki/Q656083","display_name":"Paging","level":2,"score":0.2921000123023987},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2888999879360199},{"id":"https://openalex.org/C854659","wikidata":"https://www.wikidata.org/wiki/Q1859284","display_name":"Message passing","level":2,"score":0.2822999954223633},{"id":"https://openalex.org/C2778119891","wikidata":"https://www.wikidata.org/wiki/Q477690","display_name":"CUDA","level":2,"score":0.2777999937534332},{"id":"https://openalex.org/C101722063","wikidata":"https://www.wikidata.org/wiki/Q218825","display_name":"Random access","level":2,"score":0.2773999869823456},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2761000096797943},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.2612000107765198},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2526000142097473},{"id":"https://openalex.org/C46637626","wikidata":"https://www.wikidata.org/wiki/Q6693015","display_name":"Low latency (capital markets)","level":2,"score":0.25}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2611.14848","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.14848","pdf_url":"https://arxiv.org/pdf/2611.14848","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2611.14848","doi":"https://doi.org/10.48550/arxiv.2611.14848","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.14848","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2611.14848","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.14848","pdf_url":"https://arxiv.org/pdf/2611.14848","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G6624552157","display_name":null,"funder_award_id":"W911NF-23-2-0224","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G9001310276","display_name":"NSF AI Institute for Edge Computing and Systems (Athena)","funder_award_id":"2625580","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7226008802.pdf","grobid_xml":"https://content.openalex.org/works/W7226008802.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W4391272939","https://openalex.org/W4407800405","https://openalex.org/W4414939355","https://openalex.org/W7125814462","https://openalex.org/W7159591345","https://openalex.org/W7165385035","https://openalex.org/W7165640641","https://openalex.org/W7166707533","https://openalex.org/W7167604932","https://openalex.org/W7203682669","https://openalex.org/W7213526126"],"related_works":["https://openalex.org/W7225136573","https://openalex.org/W6929720056","https://openalex.org/W4414727549","https://openalex.org/W4391272939","https://openalex.org/W7165649855","https://openalex.org/W4417085581","https://openalex.org/W7162424328","https://openalex.org/W7166741390","https://openalex.org/W4407197341","https://openalex.org/W4417065185"],"abstract_inverted_index":{"Large":[0],"language":[1],"model":[2],"(LLM)":[3],"agents":[4],"are":[5,48],"rapidly":[6],"becoming":[7],"a":[8,17,98,123],"dominant":[9],"paradigm":[10],"for":[11,100],"AI":[12],"applications.":[13],"Recently,":[14],"there":[15],"is":[16,114,203],"growing":[18],"demand":[19],"to":[20,30,62,79,115,137],"deploy":[21],"large-scale":[22],"Mixture-of-Experts":[23],"(MoE)":[24],"models":[25,106,167],"locally":[26],"on":[27,107,160],"consumer":[28,46,109,162],"PCs":[29],"balance":[31],"data":[32],"privacy,":[33],"API":[34],"costs,":[35],"and":[36,55,68,91,119,144,185],"inference":[37,102],"efficiency.":[38],"However,":[39],"unlike":[40],"datacenter":[41],"servers":[42],"with":[43,146,164,191],"abundant":[44],"DRAM,":[45],"devices":[47],"tightly":[49],"constrained":[50],"in":[51,93,182,187],"both":[52],"GPU":[53,120],"VRAM":[54,121],"host":[56,66,117],"RAM,":[57,67],"forcing":[58],"massive":[59],"MoE":[60,105,166],"weights":[61],"spill":[63],"across":[64],"VRAM,":[65],"SSD.":[69],"Under":[70],"such":[71],"spilling,":[72],"we":[73],"observe":[74],"that":[75,150,169],"existing":[76],"systems":[77],"fail":[78],"fully":[80],"utilize":[81],"this":[82,128],"memory":[83,125,133],"hierarchy,":[84],"treating":[85],"the":[86,138,197],"tiers":[87],"as":[88,122],"independent":[89],"caches":[90],"thrashing":[92],"inference.":[94],"We":[95],"present":[96],"TIDEWAY,":[97],"system":[99],"efficient":[101],"of":[103,142,180],"large":[104],"memory-constrained":[108],"devices.":[110],"Our":[111,158,200],"core":[112],"insight":[113],"manage":[116],"RAM":[118],"unified":[124],"pool.":[126],"Within":[127],"pool,":[129],"TIDEWAY":[130,170,177],"couples":[131],"phase-aware":[132],"layouts,":[134],"which":[135],"adapt":[136],"distinct":[139],"access":[140],"patterns":[141],"prefill":[143],"decode,":[145],"an":[147],"execution":[148,154],"engine":[149],"optimizes":[151],"CUDA":[152],"Graph":[153],"under":[155],"expert":[156],"paging.":[157],"evaluations":[159],"representative":[161],"hardware":[163],"frontier":[165],"demonstrate":[168],"significantly":[171],"outperforms":[172],"state-of-the-art":[173],"systems.":[174],"On":[175],"average,":[176],"achieves":[178],"speedups":[179],"1.97x":[181],"end-to-end":[183],"latency":[184],"2.40x":[186],"Time-to-First-Token":[188],"(TTFT),":[189],"along":[190],"1.61x":[192],"higher":[193],"decode":[194],"throughput":[195],"against":[196],"fastest":[198],"baseline.":[199],"source":[201],"code":[202],"publicly":[204],"available":[205],"at":[206],"https://github.com/Afterglow231/Tideway":[207],".":[208]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7226008802","counts_by_year":[],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2026-11-14T00:00:00"}
