{"id":"https://openalex.org/W4399198251","doi":"https://doi.org/10.48550/arxiv.2405.18832","title":"MoNDE: Mixture of Near-Data Experts for Large-Scale Sparse Models","display_name":"MoNDE: Mixture of Near-Data Experts for Large-Scale Sparse Models","publication_year":2024,"publication_date":"2024-05-29","ids":{"openalex":"https://openalex.org/W4399198251","doi":"https://doi.org/10.48550/arxiv.2405.18832"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.18832","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.18832","pdf_url":"https://arxiv.org/pdf/2405.18832","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.18832","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100439006","display_name":"Tae Hyun Kim","orcid":"https://orcid.org/0000-0003-1053-8958"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Kim, Taehyun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101369542","display_name":"Kwanseok Choi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choi, Kwanseok","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051294865","display_name":"Y.H. Cho","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cho, Youngmock","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003364298","display_name":"Jae Hoon Cho","orcid":"https://orcid.org/0000-0002-2243-7428"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cho, Jaehoon","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024723558","display_name":"Hyuk\u2010Jae Lee","orcid":"https://orcid.org/0000-0001-6811-9647"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Hyuk-Jae","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5114170728","display_name":"Jaewoong Sim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sim, Jaewoong","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100439006"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11980","display_name":"Human Mobility and Location-Based Analysis","score":0.9621999859809875,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11980","display_name":"Human Mobility and Location-Based Analysis","score":0.9621999859809875,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11344","display_name":"Traffic Prediction and Management Techniques","score":0.913100004196167,"subfield":{"id":"https://openalex.org/subfields/2215","display_name":"Building and Construction"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9057999849319458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.6150166988372803},{"id":"https://openalex.org/keywords/mixture-model","display_name":"Mixture model","score":0.44862276315689087},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.437140554189682},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.336117148399353},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2657397985458374},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.15099555253982544},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.08679860830307007}],"concepts":[{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.6150166988372803},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.44862276315689087},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.437140554189682},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.336117148399353},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2657397985458374},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.15099555253982544},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.08679860830307007}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.18832","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.18832","pdf_url":"https://arxiv.org/pdf/2405.18832","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.18832","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.18832","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.18832","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.18832","pdf_url":"https://arxiv.org/pdf/2405.18832","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320317879","display_name":"SK Hynix","ror":null},{"id":"https://openalex.org/F4320321292","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542"},{"id":"https://openalex.org/F4320322202","display_name":"IC Design Education Center","ror":"https://ror.org/005v57z85"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"},{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"},{"id":"https://openalex.org/F4320336769","display_name":"Institute of Engineering Research, Seoul National University","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399198251.pdf","grobid_xml":"https://content.openalex.org/works/W4399198251.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Mixture-of-Experts":[0],"(MoE)":[1],"large":[2],"language":[3],"models":[4],"(LLM)":[5],"have":[6],"memory":[7,14,75],"requirements":[8],"that":[9,43],"often":[10],"exceed":[11],"the":[12,24,51,60,64,68,73,79,86,104],"GPU":[13,25],"capacity,":[15],"requiring":[16],"costly":[17],"parameter":[18,55,106],"movement":[19,56],"from":[20],"secondary":[21],"memories":[22],"to":[23,63],"for":[26,109],"expert":[27,83],"computation.":[28],"In":[29],"this":[30],"work,":[31],"we":[32],"present":[33],"Mixture":[34],"of":[35,53,81,88],"Near-Data":[36],"Experts":[37],"(MoNDE),":[38],"a":[39],"near-data":[40],"computing":[41,67],"solution":[42],"efficiently":[44],"enables":[45,92],"MoE":[46,54,96],"LLM":[47],"inference.":[48],"MoNDE":[49,91],"reduces":[50],"volume":[52],"by":[57],"transferring":[58],"only":[59],"$\\textit{hot}$":[61],"experts":[62,71],"GPU,":[65],"while":[66],"remaining":[69],"$\\textit{cold}$":[70],"inside":[72],"host":[74],"device.":[76],"By":[77],"replacing":[78],"transfers":[80],"massive":[82],"parameters":[84],"with":[85],"ones":[87],"small":[89],"activations,":[90],"far":[93],"more":[94],"communication-efficient":[95],"inference,":[97],"thereby":[98],"resulting":[99],"in":[100],"substantial":[101],"speedups":[102],"over":[103],"existing":[105],"offloading":[107],"frameworks":[108],"both":[110],"encoder":[111],"and":[112],"decoder":[113],"operations.":[114]},"counts_by_year":[],"updated_date":"2026-02-09T09:26:11.010843","created_date":"2025-10-10T00:00:00"}