{"id":"https://openalex.org/W4416004591","doi":"https://doi.org/10.1145/3731599.3767699","title":"Pretraining LLMs at Scale: Tuning Strategies and Performance Portability.","display_name":"Pretraining LLMs at Scale: Tuning Strategies and Performance Portability.","publication_year":2025,"publication_date":"2025-11-07","ids":{"openalex":"https://openalex.org/W4416004591","doi":"https://doi.org/10.1145/3731599.3767699"},"language":null,"primary_location":{"id":"doi:10.1145/3731599.3767699","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731599.3767699","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049023670","display_name":"Adri\u00e1n Perez Di\u00e9guez","orcid":"https://orcid.org/0000-0001-7168-9050"},"institutions":[{"id":"https://openalex.org/I4210087596","display_name":"Qualcomm (United States)","ror":"https://ror.org/002zrf773","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087596"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adri\u00e1n P\u00e9rez Di\u00e9guez","raw_affiliation_strings":["Qualcomm Technologies, Inc., San Diego, USA"],"raw_orcid":"https://orcid.org/0000-0001-7168-9050","affiliations":[{"raw_affiliation_string":"Qualcomm Technologies, Inc., San Diego, USA","institution_ids":["https://openalex.org/I4210087596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120297667","display_name":"Alex Batlle Casellas","orcid":"https://orcid.org/0009-0006-1469-0358"},"institutions":[{"id":"https://openalex.org/I19268510","display_name":"Qualcomm (United Kingdom)","ror":"https://ror.org/04d3djg48","country_code":"GB","type":"company","lineage":["https://openalex.org/I19268510","https://openalex.org/I4210087596"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"\u00c0lex Batlle Casellas","raw_affiliation_strings":["Qualcomm Europe, Inc., Barcelona, Spain"],"raw_orcid":"https://orcid.org/0009-0006-1469-0358","affiliations":[{"raw_affiliation_string":"Qualcomm Europe, Inc., Barcelona, Spain","institution_ids":["https://openalex.org/I19268510"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120297668","display_name":"Aleix Torres-Camps","orcid":"https://orcid.org/0009-0007-2473-3590"},"institutions":[{"id":"https://openalex.org/I19268510","display_name":"Qualcomm (United Kingdom)","ror":"https://ror.org/04d3djg48","country_code":"GB","type":"company","lineage":["https://openalex.org/I19268510","https://openalex.org/I4210087596"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Aleix Torres-Camps","raw_affiliation_strings":["Qualcomm Europe, Inc., Barcelona, Spain"],"raw_orcid":"https://orcid.org/0009-0007-2473-3590","affiliations":[{"raw_affiliation_string":"Qualcomm Europe, Inc., Barcelona, Spain","institution_ids":["https://openalex.org/I19268510"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070190311","display_name":"Harris Teague","orcid":"https://orcid.org/0000-0002-9409-5674"},"institutions":[{"id":"https://openalex.org/I4210087596","display_name":"Qualcomm (United States)","ror":"https://ror.org/002zrf773","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087596"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Harris Teague","raw_affiliation_strings":["Qualcomm Technologies, Inc., San Diego, USA"],"raw_orcid":"https://orcid.org/0000-0002-9409-5674","affiliations":[{"raw_affiliation_string":"Qualcomm Technologies, Inc., San Diego, USA","institution_ids":["https://openalex.org/I4210087596"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011003534","display_name":"Jordi Ros-Giralt","orcid":"https://orcid.org/0000-0003-4450-609X"},"institutions":[{"id":"https://openalex.org/I19268510","display_name":"Qualcomm (United Kingdom)","ror":"https://ror.org/04d3djg48","country_code":"GB","type":"company","lineage":["https://openalex.org/I19268510","https://openalex.org/I4210087596"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jordi Ros-Giralt","raw_affiliation_strings":["Qualcomm Europe, Inc., Barcelona, Spain"],"raw_orcid":"https://orcid.org/0000-0003-4450-609X","affiliations":[{"raw_affiliation_string":"Qualcomm Europe, Inc., Barcelona, Spain","institution_ids":["https://openalex.org/I19268510"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.1551586,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1512","last_page":"1523"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.12470000237226486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.12470000237226486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.1143999993801117,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10080000013113022,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-portability","display_name":"Software portability","score":0.6572999954223633},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6484000086784363},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.602400004863739},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5184000134468079},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4699000120162964}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6582000255584717},{"id":"https://openalex.org/C63000827","wikidata":"https://www.wikidata.org/wiki/Q3080428","display_name":"Software portability","level":2,"score":0.6572999954223633},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6484000086784363},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.602400004863739},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5184000134468079},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4699000120162964},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.39160001277923584},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3824999928474426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.325300008058548},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.31360000371932983},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30469998717308044},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.27630001306533813}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3731599.3767699","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731599.3767699","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W110609005","https://openalex.org/W1790519466","https://openalex.org/W1863336885","https://openalex.org/W1964031104","https://openalex.org/W2004683477","https://openalex.org/W2016618963","https://openalex.org/W2033088400","https://openalex.org/W2037638470","https://openalex.org/W2038361169","https://openalex.org/W2061788314","https://openalex.org/W2081146773","https://openalex.org/W2084482561","https://openalex.org/W2096070062","https://openalex.org/W2099625934","https://openalex.org/W2100218206","https://openalex.org/W2107483876","https://openalex.org/W2121546953","https://openalex.org/W2167334577","https://openalex.org/W2293902057","https://openalex.org/W2747274098","https://openalex.org/W2898099374","https://openalex.org/W3087999580","https://openalex.org/W3096425133","https://openalex.org/W3124802618","https://openalex.org/W3129831491","https://openalex.org/W3131110481","https://openalex.org/W3138837712","https://openalex.org/W3197720002","https://openalex.org/W4245206864","https://openalex.org/W4286850314","https://openalex.org/W4318603216","https://openalex.org/W4367147621","https://openalex.org/W4381886086","https://openalex.org/W4382931639","https://openalex.org/W4388031315","https://openalex.org/W4400410176","https://openalex.org/W4401017732","https://openalex.org/W4402702025","https://openalex.org/W4405756397","https://openalex.org/W4406157363","https://openalex.org/W4406157795"],"related_works":[],"abstract_inverted_index":{"Training":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"at":[5,52],"scale":[6],"presents":[7],"challenges":[8],"that":[9,72],"demand":[10],"careful":[11],"co-design":[12,114],"across":[13,42],"software,":[14],"hardware,":[15],"and":[16,32,91,107],"parallelization":[17],"strategies.":[18],"In":[19],"this":[20],"work,":[21],"we":[22],"introduce":[23],"a":[24,53,110],"communication-aware":[25],"tuning":[26,106],"methodology":[27],"for":[28,104,109],"optimizing":[29],"LLM":[30,49,77,120],"pretraining,":[31],"adapt":[33],"the":[34,81,86,92,102],"performance":[35,117],"portability":[36],"metric":[37],"to":[38,60,115],"evaluate":[39],"LLM-training":[40],"efficiency":[41,118],"our":[43],"systems.":[44],"Our":[45,99],"methodology,":[46],"validated":[47],"through":[48],"pretraining":[50],"workloads":[51],"leading":[54],"global":[55],"technology":[56],"enterprise,":[57],"delivered":[58],"up":[59],"1.6x":[61],"speedup":[62],"over":[63],"default":[64,87],"configurations.":[65],"We":[66],"further":[67],"provide":[68],"six":[69],"key":[70],"insights":[71],"challenge":[73],"prevailing":[74],"assumptions":[75],"in":[76,119],"training":[78],"performance,":[79],"including":[80],"trade-offs":[82],"between":[83],"ZeRO":[84],"stages,":[85],"DeepSpeed":[88],"communication":[89],"collectives,":[90],"critical":[93],"role":[94],"of":[95],"batch":[96],"size":[97],"choices.":[98],"findings":[100],"highlight":[101],"need":[103],"platform-specific":[105],"advocate":[108],"shift":[111],"toward":[112],"end-to-end":[113],"unlock":[116],"training.":[121]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-07T00:00:00"}