{"id":"https://openalex.org/W4394780940","doi":"https://doi.org/10.48550/arxiv.2404.07413","title":"JetMoE: Reaching Llama2 Performance with 0.1M Dollars","display_name":"JetMoE: Reaching Llama2 Performance with 0.1M Dollars","publication_year":2024,"publication_date":"2024-04-11","ids":{"openalex":"https://openalex.org/W4394780940","doi":"https://doi.org/10.48550/arxiv.2404.07413"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2404.07413","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.07413","pdf_url":"https://arxiv.org/pdf/2404.07413","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2404.07413","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073742611","display_name":"Yikang Shen","orcid":"https://orcid.org/0009-0008-5263-4837","observed_orcids":["https://orcid.org/0009-0008-5263-4837","https://orcid.org/0000-0001-6836-0510"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shen, Yikang","raw_affiliation_strings":["MIT-IBM Watson AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT-IBM Watson AI Lab","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100329572","display_name":"Zhen Guo","orcid":"https://orcid.org/0000-0002-1347-3451","observed_orcids":["https://orcid.org/0000-0002-1347-3451"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Guo, Zhen","raw_affiliation_strings":["MIT EECS"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT EECS","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083555133","display_name":"Tianle Cai","orcid":"https://orcid.org/0009-0008-8644-3895","observed_orcids":["https://orcid.org/0009-0008-8644-3895"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cai, Tianle","raw_affiliation_strings":["Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052261160","display_name":"Zengyi Qin","orcid":"https://orcid.org/0000-0002-5477-9764","observed_orcids":["https://orcid.org/0000-0002-5477-9764"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qin, Zengyi","raw_affiliation_strings":["MIT EECS"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT EECS","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8598999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8598999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08309999853372574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.028999999165534973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sparse-mixture-of-experts","display_name":"sparse mixture-of-experts","score":0.7770000100135803},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6769999861717224},{"id":"https://openalex.org/keywords/training-cost-reduction","display_name":"training cost reduction","score":0.45399999618530273}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7078806161880493},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6505951881408691},{"id":"https://openalex.org/C2776650193","wikidata":"https://www.wikidata.org/wiki/Q264661","display_name":"Obstacle","level":2,"score":0.6074803471565247},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.5843675136566162},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4776279330253601},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4638998508453369},{"id":"https://openalex.org/C38858127","wikidata":"https://www.wikidata.org/wiki/Q5441228","display_name":"Feed forward","level":2,"score":0.4539484977722168},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4503747224807739},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.429257869720459},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4022432863712311},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34694957733154297},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34137576818466187},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.24142828583717346},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1298016905784607},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1206592321395874},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.10697013139724731},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C133731056","wikidata":"https://www.wikidata.org/wiki/Q4917288","display_name":"Control engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2404.07413","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.07413","pdf_url":"https://arxiv.org/pdf/2404.07413","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2404.07413","doi":"https://doi.org/10.48550/arxiv.2404.07413","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2404.07413","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2404.07413","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.07413","pdf_url":"https://arxiv.org/pdf/2404.07413","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394780940.pdf","grobid_xml":"https://content.openalex.org/works/W4394780940.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4405433133","https://openalex.org/W6891976144","https://openalex.org/W4406505516","https://openalex.org/W4390632065","https://openalex.org/W4416331037","https://openalex.org/W4392223155","https://openalex.org/W4386555464","https://openalex.org/W4417097309","https://openalex.org/W7204562970","https://openalex.org/W6891906020"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4,112,152],"achieved":[5],"remarkable":[6],"results,":[7],"but":[8],"their":[9],"increasing":[10],"resource":[11],"demand":[12],"has":[13],"become":[14],"a":[15,31],"major":[16],"obstacle":[17],"to":[18,111,130,158,172],"the":[19,57,65,71,163,179],"development":[20,164],"of":[21,99,165,181],"powerful":[22],"and":[23,48,68,101,137,143,149,175,183],"accessible":[24,182],"super-human":[25],"intelligence.":[26],"This":[27,169],"report":[28,157],"introduces":[29],"JetMoE-8B,":[30],"new":[32],"LLM":[33,78],"trained":[34],"with":[35,62],"less":[36],"than":[37,85],"$0.1":[38],"million,":[39],"using":[40,139],"1.25T":[41],"tokens":[42],"from":[43],"carefully":[44],"mixed":[45],"open-source":[46],"corpora":[47],"30,000":[49],"H100":[50],"GPU":[51],"hours.":[52],"Despite":[53],"its":[54],"low":[55],"cost,":[56],"JetMoE-8B":[58,63,88,110,133],"demonstrates":[59],"impressive":[60],"performance,":[61],"outperforming":[64],"Llama2-7B":[66],"model":[67,187],"JetMoE-8B-Chat":[69],"surpassing":[70],"Llama2-13B-Chat":[72],"model.":[73],"These":[74],"results":[75],"suggest":[76],"that":[77],"training":[79,144,147],"can":[80],"be":[81],"much":[82],"more":[83],"cost-effective":[84],"generally":[86],"thought.":[87],"is":[89,134],"based":[90],"on":[91],"an":[92],"efficient":[93,184],"Sparsely-gated":[94],"Mixture-of-Experts":[95],"(SMoE)":[96],"architecture,":[97],"composed":[98],"attention":[100],"feedforward":[102],"experts.":[103],"Both":[104],"layers":[105],"are":[106,189],"sparsely":[107],"activated,":[108],"allowing":[109],"8B":[113],"parameters":[114,148],"while":[115],"only":[116,140],"activating":[117],"2B":[118],"for":[119],"each":[120],"input":[121],"token,":[122],"reducing":[123],"inference":[124],"computation":[125],"by":[126],"about":[127],"70%":[128],"compared":[129],"Llama2-7B.":[131],"Moreover,":[132],"highly":[135],"open":[136,166],"academia-friendly,":[138],"public":[141],"datasets":[142],"code.":[145],"All":[146],"data":[150],"mixtures":[151],"been":[153],"detailed":[154],"in":[155,162,178],"this":[156],"facilitate":[159],"future":[160],"efforts":[161],"foundation":[167],"models.":[168],"transparency":[170],"aims":[171],"encourage":[173],"collaboration":[174],"further":[176],"advancements":[177],"field":[180],"LLMs.":[185],"The":[186],"weights":[188],"publicly":[190],"available":[191],"at":[192],"https://github.com/myshell-ai/JetMoE.":[193]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2024-04-13T00:00:00"}