{"id":"https://openalex.org/W4415308595","doi":"https://doi.org/10.1145/3725843.3756111","title":"Characterizing the Efficiency of Distributed Training: A Power, Performance, and Thermal Perspective","display_name":"Characterizing the Efficiency of Distributed Training: A Power, Performance, and Thermal Perspective","publication_year":2025,"publication_date":"2025-10-17","ids":{"openalex":"https://openalex.org/W4415308595","doi":"https://doi.org/10.1145/3725843.3756111"},"language":null,"primary_location":{"id":"doi:10.1145/3725843.3756111","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3725843.3756111","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3725843.3756111","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 58th IEEE/ACM International Symposium on Microarchitecture","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3725843.3756111","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116225227","display_name":"Seokjin Go","orcid":"https://orcid.org/0009-0006-9225-7728"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Seokjin Go","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":"https://orcid.org/0009-0006-9225-7728","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042175526","display_name":"Joongun Park","orcid":"https://orcid.org/0009-0000-8406-0817"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]},{"id":"https://openalex.org/I2800444561","display_name":"Atlanta Technical College","ror":"https://ror.org/01s3vfp47","country_code":"US","type":"education","lineage":["https://openalex.org/I2800444561"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joongun Park","raw_affiliation_strings":["Georgia Tech, Atlanta, USA"],"raw_orcid":"https://orcid.org/0009-0000-8406-0817","affiliations":[{"raw_affiliation_string":"Georgia Tech, Atlanta, USA","institution_ids":["https://openalex.org/I130701444","https://openalex.org/I2800444561"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Spandan More","orcid":"https://orcid.org/0009-0008-5903-2847"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Spandan More","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":"https://orcid.org/0009-0008-5903-2847","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102596877","display_name":"Hanjiang Wu","orcid":"https://orcid.org/0009-0003-3718-5272"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]},{"id":"https://openalex.org/I2800444561","display_name":"Atlanta Technical College","ror":"https://ror.org/01s3vfp47","country_code":"US","type":"education","lineage":["https://openalex.org/I2800444561"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanjiang Wu","raw_affiliation_strings":["Georgia Tech, Atlanta, USA"],"raw_orcid":"https://orcid.org/0009-0003-3718-5272","affiliations":[{"raw_affiliation_string":"Georgia Tech, Atlanta, USA","institution_ids":["https://openalex.org/I130701444","https://openalex.org/I2800444561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090421759","display_name":"I.-Hung Wang","orcid":"https://orcid.org/0000-0003-1912-5834"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Irene Wang","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":"https://orcid.org/0000-0003-1912-5834","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028394517","display_name":"Aaron Jezghani","orcid":"https://orcid.org/0000-0002-4302-4227"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaron Jezghani","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":"https://orcid.org/0000-0002-4302-4227","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034089074","display_name":"Tushar Krishna","orcid":"https://orcid.org/0000-0001-5738-6942"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tushar Krishna","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":"https://orcid.org/0000-0001-5738-6942","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089590312","display_name":"Divya Mahajan","orcid":"https://orcid.org/0009-0007-8184-0528"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Divya Mahajan","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, USA"],"raw_orcid":"https://orcid.org/0009-0007-8184-0528","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.6734,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.97952111,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"626","last_page":"642"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9871000051498413,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6110000014305115},{"id":"https://openalex.org/keywords/thermal","display_name":"Thermal","score":0.41749998927116394},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.2930000126361847},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.2689000070095062},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.23880000412464142}],"concepts":[{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6110000014305115},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4562999904155731},{"id":"https://openalex.org/C204530211","wikidata":"https://www.wikidata.org/wiki/Q752823","display_name":"Thermal","level":2,"score":0.41749998927116394},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2930000126361847},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.2689000070095062},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2538999915122986},{"id":"https://openalex.org/C39432304","wikidata":"https://www.wikidata.org/wiki/Q188847","display_name":"Environmental science","level":0,"score":0.249099999666214},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.23880000412464142},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.22840000689029694},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.21960000693798065}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3725843.3756111","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3725843.3756111","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3725843.3756111","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 58th IEEE/ACM International Symposium on Microarchitecture","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3725843.3756111","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3725843.3756111","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3725843.3756111","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 58th IEEE/ACM International Symposium on Microarchitecture","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306087","display_name":"Semiconductor Research Corporation","ror":"https://ror.org/047z4n946"},{"id":"https://openalex.org/F4320309321","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415308595.pdf","grobid_xml":"https://content.openalex.org/works/W4415308595.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W2163687928","https://openalex.org/W2261808795","https://openalex.org/W2618530766","https://openalex.org/W2765927961","https://openalex.org/W2903901007","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3096425133","https://openalex.org/W3200211247","https://openalex.org/W3204998121","https://openalex.org/W4229801112","https://openalex.org/W4242577057","https://openalex.org/W4310895557","https://openalex.org/W4312060029","https://openalex.org/W4376652719","https://openalex.org/W4380875476","https://openalex.org/W4381886086","https://openalex.org/W4387321084","https://openalex.org/W4387321093","https://openalex.org/W4387321645","https://openalex.org/W4401211718","https://openalex.org/W4404954401","https://openalex.org/W4412987318"],"related_works":[],"abstract_inverted_index":{"The":[0],"rapid":[1],"scaling":[2,104],"of":[3,16,23,39,86,204],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"has":[8],"pushed":[9],"training":[10,41,175],"workloads":[11,45],"far":[12],"beyond":[13,157],"the":[14,84,200],"limits":[15],"single-node":[17],"analysis,":[18],"demanding":[19],"a":[20,36,158],"deeper":[21],"understanding":[22],"how":[24,174],"these":[25],"models":[26,60],"behave":[27],"across":[28,42],"large-scale,":[29],"multi-GPU":[30],"systems.In":[31],"this":[32],"paper,":[33],"we":[34],"present":[35],"comprehensive":[37],"characterization":[38],"LLM":[40,206],"diverse":[43],"real-world":[44],"and":[46,52,58,68,79,92,164,186,195,202,208],"hardware":[47,75,105,196],"platforms,":[48],"including":[49],"NVIDIA":[50],"H100/H200":[51],"AMD":[53],"MI250":[54],"GPUs.We":[55],"analyze":[56],"dense":[57],"sparse":[59],"under":[61,121],"various":[62],"parallelism":[63,137],"strategies":[64],"-tensor,":[65],"pipeline,":[66,143],"data,":[67],"expert":[69],"-and":[70],"evaluate":[71,83],"their":[72],"effects":[73],"on":[74],"utilization,":[76],"power":[77,166],"consumption,":[78],"thermal":[80,170],"behavior.We":[81],"further":[82],"effectiveness":[85],"optimizations":[87],"such":[88,139],"as":[89,140],"activation":[90],"recomputation":[91],"compute-communication":[93],"overlap.Our":[94],"findings":[95],"show":[96,134],"that":[97,135,168],"performance":[98,176],"is":[99,177],"not":[100],"determined":[101],"solely":[102],"by":[103,179,190],"capacity.Scale-up":[106],"systems":[107,115,207],"with":[108,142],"fewer,":[109],"higher-memory":[110],"GPUs":[111],"can":[112],"outperform":[113],"scale-out":[114,128],"in":[116,125],"communication-bound":[117],"regimes,":[118],"but":[119],"only":[120],"carefully":[122],"tuned":[123],"configurations;":[124],"other":[126],"cases,":[127],"deployments":[129],"achieve":[130],"superior":[131],"throughput.We":[132],"also":[133],"certain":[136,159],"combinations,":[138],"tensor":[141],"lead":[144],"to":[145,149,198],"bandwidth":[146],"underutilization":[147],"due":[148],"inefficient":[150],"data":[151],"chunking,":[152],"while":[153],"increasing":[154],"microbatch":[155],"sizes":[156],"point":[160],"induces":[161],"bursty":[162],"execution":[163],"peak":[165],"excursions":[167],"worsen":[169],"throttling.These":[171],"insights":[172],"reveal":[173],"shaped":[178],"complex":[180],"interactions":[181],"between":[182],"hardware,":[183],"system":[184,194],"topology,":[185],"model":[187],"execution.We":[188],"conclude":[189],"offering":[191],"recommendations":[192],"for":[193],"design":[197],"improve":[199],"scalability":[201],"reliability":[203],"future":[205],"workloads.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":5}],"updated_date":"2026-08-11T07:18:39.950985","created_date":"2025-10-18T00:00:00"}