{"id":"https://openalex.org/W7224897913","doi":"https://doi.org/10.5281/zenodo.23811994","title":"Pyrite Phase 2: Decoupled Sparse Memory Hierarchy for 100B+ Mixture-of-Experts on Consumer Unified Memory","display_name":"Pyrite Phase 2: Decoupled Sparse Memory Hierarchy for 100B+ Mixture-of-Experts on Consumer Unified Memory","publication_year":2026,"publication_date":"2026-10-30","ids":{"openalex":"https://openalex.org/W7224897913","doi":"https://doi.org/10.5281/zenodo.23811994"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23811994","doi":"https://doi.org/10.5281/zenodo.23811994","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23811994","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23811994","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5156824417","display_name":"Masahiro Nakatsugawa","orcid":"https://orcid.org/0009-0000-3224-0103","observed_orcids":["https://orcid.org/0009-0000-3224-0103"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Nakatsugawa, Masahiro","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0000-3224-0103","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5156824417"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9570000171661377,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9570000171661377,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.013299999758601189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.004100000020116568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.9860000014305115},{"id":"https://openalex.org/keywords/unified-memory","display_name":"unified memory","score":0.9409999847412109},{"id":"https://openalex.org/keywords/consumer-grade-hardware","display_name":"consumer hardware","score":0.3619999885559082},{"id":"https://openalex.org/keywords/memory-efficiency","display_name":"memory efficiency","score":0.6480000019073486},{"id":"https://openalex.org/keywords/lru-cache","display_name":"LRU cache","score":0.8989999890327454},{"id":"https://openalex.org/keywords/on-device-large-language-models","display_name":"on-device large language models","score":0.4749999940395355}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7382000088691711},{"id":"https://openalex.org/C81081738","wikidata":"https://www.wikidata.org/wiki/Q55542","display_name":"Lossless compression","level":3,"score":0.6014999747276306},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.5389000177383423},{"id":"https://openalex.org/C136085584","wikidata":"https://www.wikidata.org/wiki/Q910289","display_name":"Overlay","level":2,"score":0.487199991941452},{"id":"https://openalex.org/C2778100165","wikidata":"https://www.wikidata.org/wiki/Q1589327","display_name":"Memory hierarchy","level":3,"score":0.47530001401901245},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4749000072479248},{"id":"https://openalex.org/C63511323","wikidata":"https://www.wikidata.org/wiki/Q908936","display_name":"Interleaved memory","level":4,"score":0.4666000008583069},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.4478999972343445},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4368000030517578},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4090000092983246},{"id":"https://openalex.org/C2775896111","wikidata":"https://www.wikidata.org/wiki/Q642560","display_name":"Router","level":2,"score":0.3986000120639801},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.391400009393692},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.36719998717308044},{"id":"https://openalex.org/C149810388","wikidata":"https://www.wikidata.org/wiki/Q5374873","display_name":"Emulation","level":2,"score":0.3605000078678131},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.3431999981403351},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.33730000257492065},{"id":"https://openalex.org/C68043766","wikidata":"https://www.wikidata.org/wiki/Q267416","display_name":"Static random-access memory","level":2,"score":0.33340001106262207},{"id":"https://openalex.org/C82687282","wikidata":"https://www.wikidata.org/wiki/Q66221","display_name":"Auxiliary memory","level":2,"score":0.32839998602867126},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3100000023841858},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3059000074863434},{"id":"https://openalex.org/C88196245","wikidata":"https://www.wikidata.org/wiki/Q8034984","display_name":"Working set","level":2,"score":0.30559998750686646},{"id":"https://openalex.org/C7366592","wikidata":"https://www.wikidata.org/wiki/Q1255620","display_name":"Dram","level":2,"score":0.30469998717308044},{"id":"https://openalex.org/C56086750","wikidata":"https://www.wikidata.org/wiki/Q6042592","display_name":"Integer programming","level":2,"score":0.30379998683929443},{"id":"https://openalex.org/C57863822","wikidata":"https://www.wikidata.org/wiki/Q905488","display_name":"Flat memory model","level":4,"score":0.3028999865055084},{"id":"https://openalex.org/C177950962","wikidata":"https://www.wikidata.org/wiki/Q10997658","display_name":"Non-volatile memory","level":2,"score":0.29919999837875366},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.29820001125335693},{"id":"https://openalex.org/C93446704","wikidata":"https://www.wikidata.org/wiki/Q449328","display_name":"Registered memory","level":3,"score":0.29440000653266907},{"id":"https://openalex.org/C2781357197","wikidata":"https://www.wikidata.org/wiki/Q5757597","display_name":"High memory","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.2888000011444092},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C74426580","wikidata":"https://www.wikidata.org/wiki/Q719484","display_name":"Memory map","level":3,"score":0.2741999924182892},{"id":"https://openalex.org/C64142963","wikidata":"https://www.wikidata.org/wiki/Q1153902","display_name":"Phase-change memory","level":3,"score":0.2728999853134155},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.27219998836517334},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.265500009059906},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.2653000056743622},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.2653000056743622},{"id":"https://openalex.org/C97137487","wikidata":"https://www.wikidata.org/wiki/Q729138","display_name":"Integer (computer science)","level":2,"score":0.2644999921321869},{"id":"https://openalex.org/C195053848","wikidata":"https://www.wikidata.org/wiki/Q7894141","display_name":"Universal memory","level":5,"score":0.2549000084400177},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.25200000405311584},{"id":"https://openalex.org/C76399640","wikidata":"https://www.wikidata.org/wiki/Q189401","display_name":"Virtual memory","level":4,"score":0.251800000667572}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.5281/zenodo.23811994","doi":"https://doi.org/10.5281/zenodo.23811994","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23811994","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23811995","doi":"https://doi.org/10.5281/zenodo.23811995","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23811995","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23812304","doi":"https://doi.org/10.5281/zenodo.23812304","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23812304","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23838797","doi":"https://doi.org/10.5281/zenodo.23838797","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23838797","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23811994","doi":"https://doi.org/10.5281/zenodo.23811994","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23811994","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7171240696","https://openalex.org/W7172106603","https://openalex.org/W7167258015","https://openalex.org/W7224814539","https://openalex.org/W7169813043","https://openalex.org/W7150801976","https://openalex.org/W7161018561","https://openalex.org/W7170262522","https://openalex.org/W7155196756","https://openalex.org/W7213292366"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"architectures":[2],"such":[3],"as":[4],"Mixtral-8x22B":[5,159],"(141B":[6],"parameters)":[7],"and":[8,80,86,104,205],"DeepSeek-Coder-V2-Lite":[9,127],"(15.7B":[10],"parameters,":[11],"64":[12],"experts)":[13],"achieve":[14],"state-of-the-art":[15],"capability,":[16],"yet":[17],"their":[18],"massive":[19],"footprint":[20],"(70":[21],"GB":[22,39,42,123,131,134,162,165,171],"to":[23,30,40,132,142,163],"280":[24],"GB)":[25],"has":[26],"restricted":[27],"local":[28],"deployment":[29],"high-end":[31],"multi-GPU":[32],"clusters":[33],"or":[34],"workstation-class":[35],"hardware":[36],"with":[37],"128":[38],"192":[41],"of":[43,135,155,166,172,184],"unified":[44],"memory.":[45],"We":[46],"present":[47],"Pyrite":[48],"Phase":[49],"2":[50],"(MoE-Stream),":[51],"an":[52,105],"execution":[53,154,204],"architecture":[54],"that":[55,189],"decouples":[56],"the":[57,70,149,156,181],"model's":[58],"static":[59],"resident":[60],"working":[61,72],"set":[62,73],"from":[63,93],"dynamic":[64],"routed":[65,90],"feed-forward":[66],"experts.":[67],"By":[68],"pinning":[69],"non-MoE":[71],"(Multi-Head":[74],"Latent":[75],"Attention,":[76],"LayerNorms,":[77],"Router":[78],"Gates,":[79],"Token":[81],"Embeddings)":[82],"into":[83],"Unified":[84],"Memory":[85],"streaming":[87],"active":[88],"Top-K":[89],"experts":[91],"on-demand":[92],"NVMe":[94],"SSD":[95],"storage":[96],"via":[97],"zero-copy":[98],"linear":[99],"byte-slicing":[100],"(68":[101],"microseconds":[102],"latency)":[103],"in-memory":[106],"Least-Recently-Used":[107],"(LRU)":[108],"cache,":[109],"MoE-Stream":[110],"achieves":[111],"unprecedented":[112],"memory":[113,144,174],"efficiency":[114],"on":[115],"standard":[116],"consumer":[117,198],"hardware.":[118],"On":[119],"a":[120],"single":[121],"16":[122],"Apple":[124],"M2":[125],"Pro,":[126],"operates":[128],"within":[129,160],"1.55":[130],"2.01":[133],"RAM":[136],"at":[137],"2.83":[138],"tokens/s":[139],"(an":[140],"83%":[141],"93%":[143],"reduction).":[145],"Furthermore,":[146],"we":[147,179],"demonstrate":[148],"successful":[150],"56-layer":[151],"forward":[152],"pass":[153],"141-Billion":[157],"parameter":[158],"2.84":[161],"3.61":[164],"RAM,":[167],"leaving":[168],"over":[169],"12":[170],"physical":[173],"available":[175],"for":[176,197],"multitasking.":[177],"Finally,":[178],"discuss":[180],"theoretical":[182],"implications":[183],"this":[185],"decoupled":[186],"hierarchy,":[187],"showing":[188],"discrete":[190],"integer":[191],"quantization":[192],"is":[193],"no":[194],"longer":[195],"mandatory":[196],"deployment,":[199],"unlocking":[200],"true":[201],"unquantized":[202],"BF16":[203],"lossless":[206],"floating-point":[207],"streaming.":[208]},"counts_by_year":[],"updated_date":"2026-11-01T11:40:01.658920","created_date":"2026-10-31T00:00:00"}