{"id":"https://openalex.org/W4396913718","doi":"https://doi.org/10.48550/arxiv.2405.07518","title":"SambaNova SN40L: Scaling the AI Memory Wall with Dataflow and Composition of Experts","display_name":"SambaNova SN40L: Scaling the AI Memory Wall with Dataflow and Composition of Experts","publication_year":2024,"publication_date":"2024-05-13","ids":{"openalex":"https://openalex.org/W4396913718","doi":"https://doi.org/10.48550/arxiv.2405.07518"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.07518","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.07518","pdf_url":"https://arxiv.org/pdf/2405.07518","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.07518","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111746097","display_name":"Raghu Prabhakar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Prabhakar, Raghu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065493508","display_name":"Ram Sivaramakrishnan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sivaramakrishnan, Ram","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007355469","display_name":"Darshan Gandhi","orcid":"https://orcid.org/0000-0002-7079-450X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gandhi, Darshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100928662","display_name":"Yun Du","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Yun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112914114","display_name":"Mingran Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Mingran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053270215","display_name":"Xiangyu Song","orcid":"https://orcid.org/0000-0002-5550-6354"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Xiangyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101503011","display_name":"Kejie Zhang","orcid":"https://orcid.org/0000-0001-9783-4412"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kejie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089291480","display_name":"Tianren Gao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Tianren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048145208","display_name":"Angela Yee\u2010Moon Wang","orcid":"https://orcid.org/0000-0003-2508-7117"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Angela","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073949362","display_name":"Karen Li","orcid":"https://orcid.org/0000-0002-0261-743X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Karen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076742208","display_name":"Yongning Sheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sheng, Yongning","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098316663","display_name":"Joshua Brot","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brot, Joshua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012419400","display_name":"\u0414.\u041d. \u0421\u043e\u043a\u043e\u043b\u043e\u0432","orcid":"https://orcid.org/0000-0003-2411-0143"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sokolov, Denis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081444766","display_name":"Apurv Vivek","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vivek, Apurv","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100863254","display_name":"Calvin Leung","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leung, Calvin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024710409","display_name":"Arjun Sabnis","orcid":"https://orcid.org/0009-0007-5699-6117"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sabnis, Arjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100928605","display_name":"Jiayu Bai","orcid":"https://orcid.org/0000-0003-3336-9464"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bai, Jiayu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033875105","display_name":"Tuowen Zhao","orcid":"https://orcid.org/0000-0002-6347-7135"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Tuowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057532402","display_name":"Mark Gottscho","orcid":"https://orcid.org/0000-0001-8370-4158"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gottscho, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039125300","display_name":"David Jackson","orcid":"https://orcid.org/0000-0002-1975-8132"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jackson, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034846342","display_name":"Mark Luttrell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luttrell, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103075809","display_name":"Manish Shah","orcid":"https://orcid.org/0000-0002-6653-676X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shah, Manish K.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044572683","display_name":"Edison Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Edison","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062351534","display_name":"Kaizhao Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Kaizhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034858834","display_name":"Swayambhoo Jain","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jain, Swayambhoo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090396089","display_name":"Urmish Thakker","orcid":"https://orcid.org/0000-0002-0515-9155"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thakker, Urmish","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062027635","display_name":"Dawei Huang","orcid":"https://orcid.org/0000-0002-4496-2354"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Dawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045706635","display_name":"Sumti Jairath","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jairath, Sumti","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103616447","display_name":"Kevin J. Brown","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brown, Kevin J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5023857198","display_name":"Kunle Olukotun","orcid":"https://orcid.org/0000-0002-8779-0636"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Olukotun, Kunle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6525999903678894,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6525999903678894,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dataflow","display_name":"Dataflow","score":0.9148526191711426},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.6683506965637207},{"id":"https://openalex.org/keywords/composition","display_name":"Composition (language)","score":0.6021493077278137},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5688188672065735},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3615036606788635},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.3387426733970642},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.32652997970581055},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.15083736181259155},{"id":"https://openalex.org/keywords/art","display_name":"Art","score":0.096751868724823},{"id":"https://openalex.org/keywords/literature","display_name":"Literature","score":0.05750912427902222},{"id":"https://openalex.org/keywords/geometry","display_name":"Geometry","score":0.0539153516292572}],"concepts":[{"id":"https://openalex.org/C96324660","wikidata":"https://www.wikidata.org/wiki/Q205446","display_name":"Dataflow","level":2,"score":0.9148526191711426},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6683506965637207},{"id":"https://openalex.org/C40231798","wikidata":"https://www.wikidata.org/wiki/Q1333743","display_name":"Composition (language)","level":2,"score":0.6021493077278137},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5688188672065735},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3615036606788635},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.3387426733970642},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.32652997970581055},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15083736181259155},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.096751868724823},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.05750912427902222},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0539153516292572}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.07518","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.07518","pdf_url":"https://arxiv.org/pdf/2405.07518","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.07518","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.07518","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.07518","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.07518","pdf_url":"https://arxiv.org/pdf/2405.07518","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4396913718.pdf","grobid_xml":"https://content.openalex.org/works/W4396913718.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2293118914","https://openalex.org/W2998381397","https://openalex.org/W4236419692","https://openalex.org/W3167919718","https://openalex.org/W4251718783","https://openalex.org/W2171015181","https://openalex.org/W4239447582","https://openalex.org/W1484403103","https://openalex.org/W2521947294","https://openalex.org/W2907307640"],"abstract_inverted_index":{"Monolithic":[0],"large":[1,103],"language":[2],"models":[3,86,106],"(LLMs)":[4],"like":[5],"GPT-4":[6],"have":[7,40,87],"paved":[8],"the":[9,62,135,157,238],"way":[10],"for":[11,174,234],"modern":[12,37],"generative":[13],"AI":[14,38,136],"applications.":[15,179],"Training,":[16],"serving,":[17],"and":[18,28,64,68,99,129,148,177,194,205,258,269],"maintaining":[19],"monolithic":[20],"LLMs":[21],"at":[22],"scale,":[23],"however,":[24],"remains":[25],"prohibitively":[26,110],"expensive":[27,111],"challenging.":[29],"The":[30,180],"disproportionate":[31],"increase":[32],"in":[33],"compute-to-memory":[34],"ratio":[35],"of":[36,52,66,105,263],"accelerators":[39],"created":[41],"a":[42,102,130,142,149,165,183,266,272],"memory":[43,132,137,186],"wall,":[44],"necessitating":[45],"new":[46,184],"methods":[47],"to":[48,97,216,247,256],"deploy":[49,154],"AI.":[50],"Composition":[51],"Experts":[53],"(CoE)":[54],"is":[55],"an":[56,228,260],"alternative":[57],"modular":[58],"approach":[59,72],"that":[60,170,233],"lowers":[61],"cost":[63],"complexity":[65],"training":[67,178],"serving.":[69],"However,":[70],"this":[71,120],"presents":[73],"two":[74],"key":[75],"challenges":[76],"when":[77,114],"using":[78],"conventional":[79],"hardware:":[80],"(1)":[81],"without":[82],"fused":[83],"operations,":[84],"smaller":[85],"lower":[88],"operational":[89],"intensity,":[90],"which":[91],"makes":[92],"high":[93],"utilization":[94],"more":[95],"challenging":[96],"achieve;":[98],"(2)":[100],"hosting":[101],"number":[104],"can":[107],"be":[108],"either":[109],"or":[112],"slow":[113],"dynamically":[115],"switching":[116,252],"between":[117],"them.":[118],"In":[119],"paper,":[121],"we":[122],"describe":[123,140],"how":[124],"combining":[125],"CoE,":[126],"streaming":[127],"dataflow,":[128],"three-tier":[131,185],"system":[133,144,187],"scales":[134],"wall.":[138],"We":[139,153,210,231],"Samba-CoE,":[141],"CoE":[143,235],"with":[145,188,227],"150":[146],"experts":[147],"trillion":[150],"total":[151],"parameters.":[152],"Samba-CoE":[155],"on":[156,218,222],"SambaNova":[158],"SN40L":[159],"Reconfigurable":[160],"Dataflow":[161],"Unit":[162],"(RDU)":[163],"-":[164],"commercial":[166],"dataflow":[167],"accelerator":[168],"architecture":[169],"has":[171],"been":[172],"co-designed":[173],"enterprise":[175],"inference":[176,236],"chip":[181],"introduces":[182],"on-chip":[189],"distributed":[190],"SRAM,":[191],"on-package":[192],"HBM,":[193],"off-package":[195],"DDR":[196],"DRAM.":[197],"A":[198],"dedicated":[199],"inter-RDU":[200],"network":[201],"enables":[202],"scaling":[203],"up":[204,246,250],"out":[206],"over":[207,265,271],"multiple":[208],"sockets.":[209],"demonstrate":[211],"speedups":[212],"ranging":[213],"from":[214],"2$\\times$":[215],"13$\\times$":[217],"various":[219],"benchmarks":[220],"running":[221],"eight":[223],"RDU":[224,240],"sockets":[225],"compared":[226],"unfused":[229],"baseline.":[230],"show":[232],"deployments,":[237],"8-socket":[239],"Node":[241],"reduces":[242],"machine":[243],"footprint":[244],"by":[245,254],"19$\\times$,":[248],"speeds":[249],"model":[251],"time":[253],"15$\\times$":[255],"31$\\times$,":[257],"achieves":[259],"overall":[261],"speedup":[262],"3.7$\\times$":[264],"DGX":[267,273],"H100":[268],"6.6$\\times$":[270],"A100.":[274]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2024-12-04T00:00:00"}