{"id":"https://openalex.org/W7223990264","doi":"https://doi.org/10.48550/arxiv.2610.34721","title":"Prefetch-Steered Routing: Deterministic Cache-Aware Expert Selection for Offloaded Mixture-of-Experts Decoding","display_name":"Prefetch-Steered Routing: Deterministic Cache-Aware Expert Selection for Offloaded Mixture-of-Experts Decoding","publication_year":2026,"publication_date":"2026-10-26","ids":{"openalex":"https://openalex.org/W7223990264","doi":"https://doi.org/10.48550/arxiv.2610.34721"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.34721","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.34721","pdf_url":"https://arxiv.org/pdf/2610.34721","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.34721","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5159899847","display_name":"Siming Huang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Huang, Siming","raw_affiliation_strings":["The Hong Kong University of Science and Technology (Guangzhou) , Guangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology (Guangzhou) , Guangzhou , China","institution_ids":["https://openalex.org/I4405261251"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5159899847"],"corresponding_institution_ids":["https://openalex.org/I4405261251"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9702000021934509,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9702000021934509,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.007400000002235174,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.003599999938160181,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.8080000281333923},{"id":"https://openalex.org/keywords/prefetching","display_name":"prefetching","score":0.7670000195503235},{"id":"https://openalex.org/keywords/expert-selection","display_name":"expert selection","score":0.800000011920929},{"id":"https://openalex.org/keywords/decoding-delay","display_name":"decoding latency","score":0.531000018119812},{"id":"https://openalex.org/keywords/gpu-memory-constraints","display_name":"GPU memory constraints","score":0.5370000004768372},{"id":"https://openalex.org/keywords/speculative-decoding","display_name":"speculative decoding","score":0.7089999914169312},{"id":"https://openalex.org/keywords/lru-cache","display_name":"LRU cache","score":0.35199999809265137}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7354999780654907},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.6388000249862671},{"id":"https://openalex.org/C2775896111","wikidata":"https://www.wikidata.org/wiki/Q642560","display_name":"Router","level":2,"score":0.5439000129699707},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.4729999899864197},{"id":"https://openalex.org/C2779151265","wikidata":"https://www.wikidata.org/wiki/Q1156791","display_name":"Copying","level":2,"score":0.4544999897480011},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.445499986410141},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.40869998931884766},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.40619999170303345},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.4016000032424927},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3528999984264374},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.3488999903202057},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.34279999136924744},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.33320000767707825},{"id":"https://openalex.org/C162262903","wikidata":"https://www.wikidata.org/wiki/Q343527","display_name":"Allocator","level":2,"score":0.31459999084472656},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.3061000108718872},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.296099990606308},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.2930999994277954},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.2890999913215637},{"id":"https://openalex.org/C115537543","wikidata":"https://www.wikidata.org/wiki/Q165596","display_name":"Cache","level":2,"score":0.2842999994754791},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.27900001406669617},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C176181172","wikidata":"https://www.wikidata.org/wiki/Q3490301","display_name":"Nondeterministic algorithm","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2660999894142151},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2606000006198883}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.34721","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.34721","pdf_url":"https://arxiv.org/pdf/2610.34721","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.34721","doi":"https://doi.org/10.48550/arxiv.2610.34721","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.34721","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.34721","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.34721","pdf_url":"https://arxiv.org/pdf/2610.34721","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7223990264.pdf","grobid_xml":"https://content.openalex.org/works/W7223990264.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7211883749","https://openalex.org/W7203928089","https://openalex.org/W7171708010","https://openalex.org/W4404308452","https://openalex.org/W4416436469","https://openalex.org/W4416238395","https://openalex.org/W7140083998","https://openalex.org/W7167030261","https://openalex.org/W7162524418","https://openalex.org/W7213372238"],"abstract_inverted_index":{"When":[0],"a":[1,49,71,227,235],"Mixture-of-Experts":[2],"(MoE)":[3],"model":[4],"does":[5,52],"not":[6,53,208,214],"fit":[7],"in":[8],"GPU":[9,252],"memory,":[10],"batch-1":[11],"decoding":[12,217],"is":[13,164],"dominated":[14],"by":[15,27,82],"copying":[16],"expert":[17,220],"weights":[18],"from":[19],"host":[20],"memory.":[21],"Cache-aware":[22],"routing":[23,66,68,117,169],"reduces":[24],"these":[25],"copies":[26,94,191],"biasing":[28],"the":[29,41,60,83,124,152,177,204,255],"router":[30,86],"toward":[31],"resident":[32,106],"experts,":[33],"but":[34,207,233],"combined":[35],"with":[36,100,118,128,170,218,226,234,248],"asynchronous":[37],"prefetching":[38],"it":[39,240],"makes":[40],"model's":[42],"outputs":[43],"depend":[44],"on":[45,70,107,151,156,183,203,231,238,254],"transfer":[46],"timing,":[47],"so":[48],"harness":[50],"that":[51,55],"simulate":[54],"timing":[56],"no":[57],"longer":[58],"evaluates":[59],"deployed":[61],"model.":[62],"We":[63],"propose":[64],"prefetch-steered":[65],"(PSR):":[67],"conditions":[69],"deterministic":[72],"logical":[73],"cache,":[74],"into":[75],"which":[76],"at":[77,158,172,176,249],"most":[78,250],"n":[79],"experts":[80,105],"predicted":[81],"next":[84],"layer's":[85],"are":[87,192],"inserted":[88],"per":[89],"layer":[90],"and":[91,102,120,134,141,154],"token;":[92],"physical":[93],"affect":[95],"only":[96],"latency.":[97],"On":[98,161,188],"OLMoE-1B-7B":[99],"50%":[101,159],"25%":[103],"of":[104,132,179,185],"an":[108],"RTX":[109],"4090,":[110],"PSR":[111,163,197,225],"decodes":[112],"2.2-2.3x":[113],"faster":[114,122,166,243],"than":[115,123,167,224,244],"exact":[116,168],"LRU":[119,171],"25-30%":[121],"Cache-Prior":[125,149,202],"operating":[126],"point,":[127],"mean":[129],"benchmark":[130],"differences":[131],"+0.1":[133],"+0.5":[135],"points":[136],"(95%":[137],"lower":[138],"bounds":[139],"-0.8":[140],"-0.3).":[142],"In":[143],"greedy":[144],"generation":[145],"its":[146,199,209,251],"lead":[147],"over":[148,201],"depends":[150],"workload":[153],"reverses":[155],"HumanEval":[157],"residency.":[160],"Moonlight-16B-A3B,":[162],"1.9x":[165],"37.5%":[173],"residency":[174],"and,":[175],"latency":[178],"Cache-Prior,":[180],"significantly":[181],"better":[182],"three":[184],"eight":[186],"benchmarks.":[187],"Qwen3.6-35B-A3B,":[189],"where":[190],"cheap":[193],"relative":[194],"to":[195],"computation,":[196],"keeps":[198],"advantage":[200],"likelihood":[205],"benchmarks":[206],"speed.":[210],"Distillation-trained":[211],"gates":[212],"did":[213],"help.":[215],"Speculative":[216],"block-level":[219],"selection":[221],"was":[222,241],"slower":[223],"1B":[228],"autoregressive":[229,246],"drafter":[230,237],"OLMoE,":[232],"block":[236],"Qwen3.6":[239],"2.1-2.9x":[242],"any":[245],"configuration":[247],"memory":[253],"64":[256],"prompts":[257],"we":[258],"timed.":[259]},"counts_by_year":[],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2026-10-28T00:00:00"}