{"id":"https://openalex.org/W4404386171","doi":"https://doi.org/10.1145/3698038.3698535","title":"Distributed Training of Large Language Models on AWS Trainium","display_name":"Distributed Training of Large Language Models on AWS Trainium","publication_year":2024,"publication_date":"2024-11-14","ids":{"openalex":"https://openalex.org/W4404386171","doi":"https://doi.org/10.1145/3698038.3698535"},"language":"en","primary_location":{"id":"doi:10.1145/3698038.3698535","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3698038.3698535","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Symposium on Cloud Computing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3698038.3698535","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020914269","display_name":"Xinwei Fu","orcid":"https://orcid.org/0009-0004-7822-5450"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Xinwei Fu","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029095203","display_name":"Zhen Zhang","orcid":"https://orcid.org/0000-0002-0164-0849"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhen Zhang","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053817404","display_name":"Haozheng Fan","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haozheng Fan","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085557190","display_name":"Guangtai Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Guangtai Huang","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114652073","display_name":"Mohammad El-Shabani","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammad El-Shabani","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074836221","display_name":"Randy Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Randy Huang","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008777481","display_name":"Rahul Solanki","orcid":"https://orcid.org/0009-0004-8176-4594"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rahul Solanki","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113264325","display_name":"Fei Wu","orcid":"https://orcid.org/0009-0000-7111-8047"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fei Wu","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091922932","display_name":"Ron Diamant","orcid":"https://orcid.org/0000-0002-6669-5250"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ron Diamant","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101638214","display_name":"Yida Wang","orcid":"https://orcid.org/0000-0001-8165-840X"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yida Wang","raw_affiliation_strings":["Amazon Web Services"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services","institution_ids":["https://openalex.org/I1311688040"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5020914269"],"corresponding_institution_ids":["https://openalex.org/I1311688040"],"apc_list":null,"apc_paid":null,"fwci":3.4752,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.93617759,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"961","last_page":"976"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9593999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9326000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7913906574249268},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6378552913665771},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4125271439552307},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3647245168685913},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3422354757785797}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7913906574249268},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6378552913665771},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4125271439552307},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3647245168685913},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3422354757785797},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3698038.3698535","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3698038.3698535","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Symposium on Cloud Computing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3698038.3698535","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3698038.3698535","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Symposium on Cloud Computing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.4399999976158142}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W2325556328","https://openalex.org/W2442974303","https://openalex.org/W2540279855","https://openalex.org/W2606722458","https://openalex.org/W2951882630","https://openalex.org/W2981758446","https://openalex.org/W3034107927","https://openalex.org/W3081168214","https://openalex.org/W3129831491","https://openalex.org/W3204998121","https://openalex.org/W4224308101","https://openalex.org/W4226199141","https://openalex.org/W4281758439","https://openalex.org/W4299828299","https://openalex.org/W4312983671","https://openalex.org/W4380874786","https://openalex.org/W4386709668","https://openalex.org/W4386768656","https://openalex.org/W4387302750","https://openalex.org/W4394998532","https://openalex.org/W4394998694","https://openalex.org/W6810081322"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W230091440","https://openalex.org/W2390279801","https://openalex.org/W2233261550","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Large":[0],"language":[1,110],"models":[2,111],"(LLMs)":[3],"are":[4,78,199],"ubiquitously":[5],"powerful":[6],"but":[7],"prohibitively":[8],"expensive":[9],"to":[10,52,70,133,154,163,172],"train,":[11],"often":[12],"requiring":[13],"thousands":[14],"of":[15,24,91,99,108,122],"compute":[16],"devices,":[17],"typically":[18],"GPUs.":[19,83],"To":[20],"reduce":[21],"the":[22,34,74,87,92,100,146],"cost":[23,68,165],"training":[25,63,107,125,184],"LLMs":[26,198],"for":[27,183],"customers,":[28],"Amazon":[29,35],"Web":[30],"Services":[31],"(AWS)":[32],"launched":[33],"EC2":[36],"trn1":[37,57,143],"instances,":[38,76,144],"powered":[39,79],"by":[40,80],"AWS":[41,101,113,178],"Trainium,":[42],"an":[43,50],"Amazon's":[44],"homegrown":[45],"deep":[46],"learning":[47],"accelerator,":[48],"as":[49,137],"alternative":[51],"distributed":[53,106,124],"LLM":[54,62],"training.":[55],"The":[56],"instances":[58],"provide":[59],"a":[60,66,97,120,176],"high-performance":[61,192],"solution":[64],"at":[65],"lower":[67,160],"compared":[69,171],"their":[71],"GPU-based":[72],"counterpart,":[73],"p4d":[75],"which":[77,104],"Nvidia":[81],"A100":[82],"This":[84],"paper":[85],"describes":[86],"design":[88],"and":[89,130,187,193],"development":[90],"Neuron":[93,102,115],"Distributed":[94,116],"Training":[95,117],"Library,":[96],"component":[98],"SDK,":[103],"enables":[105],"large":[109],"on":[112],"Trainium.":[114],"Library":[118],"supports":[119],"variety":[121],"existing":[123],"techniques":[126],"with":[127],"unified":[128],"interfaces,":[129],"provides":[131],"features":[132],"address":[134],"trn1-specific":[135],"challenges":[136],"well.":[138],"Our":[139],"evaluation":[140],"shows":[141],"that":[142],"specifically":[145],"trn1.32xlarge,":[147],"achieve":[148],"better":[149],"or":[150],"comparable":[151],"performance":[152],"(up":[153,162],"24.6%":[155],"improvement)":[156],"while":[157],"offering":[158],"significant":[159],"costs":[161],"46.3%":[164],"saving)":[166],"in":[167],"selected":[168],"workloads":[169],"when":[170],"p4d.24xlarge":[173],"instances.":[174],"As":[175],"result,":[177],"Trainium":[179],"has":[180],"been":[181],"adopted":[182],"numerous":[185],"external":[186],"internal":[188],"models,":[189],"showcasing":[190],"its":[191],"cost-effectiveness.":[194],"Several":[195],"supported":[196],"open-source":[197],"accessible":[200],"via":[201],"HuggingFace":[202],"Optimum":[203],"Neuron.":[204]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}