{"id":"https://openalex.org/W7123346683","doi":"https://doi.org/10.1002/cpe.70500","title":"Communication Frequency in Megatron\u2010LM: Experimental Insights Applied to Heterogeneous Distributed Training Time Prediction","display_name":"Communication Frequency in Megatron\u2010LM: Experimental Insights Applied to Heterogeneous Distributed Training Time Prediction","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7123346683","doi":"https://doi.org/10.1002/cpe.70500"},"language":"en","primary_location":{"id":"doi:10.1002/cpe.70500","is_oa":true,"landing_page_url":"https://doi.org/10.1002/cpe.70500","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cpe.70500","source":{"id":"https://openalex.org/S11065456","display_name":"Concurrency and Computation Practice and Experience","issn_l":"1532-0626","issn":["1532-0626","1532-0634"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Concurrency and Computation: Practice and Experience","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cpe.70500","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"HaoRan Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"HaoRan Zhang","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108996780","display_name":"Yanzhao Feng","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yanzhao Feng","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":"https://orcid.org/0009-0009-5970-0638","affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122879479","display_name":"Zhengwei Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhengwei Chen","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122849650","display_name":"Yutong Tian","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yutong Tian","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122895247","display_name":"Xiaoli Zheng","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoli Zheng","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014006906","display_name":"Cong Liu","orcid":"https://orcid.org/0009-0009-0184-5965"},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cong Liu","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102751310","display_name":"Sheng Wang","orcid":"https://orcid.org/0000-0001-8964-511X"},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sheng Wang","raw_affiliation_strings":["China Mobile Research Institute  Beijing China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Research Institute  Beijing China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122863360","display_name":"Jie Ren","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Ren","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041879346","display_name":"Yucong Li","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yucong Li","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5122908601","display_name":"Rui Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Zhu","raw_affiliation_strings":["China Mobile Qilu Innovation Research Institute  Shandong China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile Qilu Innovation Research Institute  Shandong China","institution_ids":["https://openalex.org/I180662265"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5108996780"],"corresponding_institution_ids":["https://openalex.org/I180662265"],"apc_list":{"value":4740,"currency":"USD","value_usd":4740},"apc_paid":{"value":4740,"currency":"USD","value_usd":4740},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.05185812,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"38","issue":"2","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.25279998779296875,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.25279998779296875,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.10980000346899033,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.052400000393390656,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/symmetric-multiprocessor-system","display_name":"Symmetric multiprocessor system","score":0.6301000118255615},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5073000192642212},{"id":"https://openalex.org/keywords/computational-resource","display_name":"Computational resource","score":0.4797999858856201},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.4733999967575073},{"id":"https://openalex.org/keywords/heterogeneous-network","display_name":"Heterogeneous network","score":0.45570001006126404},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.43380001187324524},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.4325999915599823},{"id":"https://openalex.org/keywords/computational-model","display_name":"Computational model","score":0.38679999113082886},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.36959999799728394}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8952999711036682},{"id":"https://openalex.org/C172430144","wikidata":"https://www.wikidata.org/wiki/Q17111997","display_name":"Symmetric multiprocessor system","level":2,"score":0.6301000118255615},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5756999850273132},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5073000192642212},{"id":"https://openalex.org/C127964446","wikidata":"https://www.wikidata.org/wiki/Q1092142","display_name":"Computational resource","level":3,"score":0.4797999858856201},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.4733999967575073},{"id":"https://openalex.org/C158207573","wikidata":"https://www.wikidata.org/wiki/Q5747224","display_name":"Heterogeneous network","level":4,"score":0.45570001006126404},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.43380001187324524},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.4325999915599823},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.38679999113082886},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.36959999799728394},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.359499990940094},{"id":"https://openalex.org/C29202148","wikidata":"https://www.wikidata.org/wiki/Q287260","display_name":"Resource allocation","level":2,"score":0.35850000381469727},{"id":"https://openalex.org/C2777115002","wikidata":"https://www.wikidata.org/wiki/Q7168246","display_name":"Performance prediction","level":2,"score":0.35420000553131104},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.33070001006126404},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3228999972343445},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3061999976634979},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.29330000281333923},{"id":"https://openalex.org/C45804977","wikidata":"https://www.wikidata.org/wiki/Q7239673","display_name":"Predictive modelling","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C106515295","wikidata":"https://www.wikidata.org/wiki/Q26806595","display_name":"Parallel processing","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.27630001306533813},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2759000062942505},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.2705000042915344},{"id":"https://openalex.org/C120373497","wikidata":"https://www.wikidata.org/wiki/Q1087987","display_name":"Parallel algorithm","level":2,"score":0.2700999975204468},{"id":"https://openalex.org/C12269588","wikidata":"https://www.wikidata.org/wiki/Q132364","display_name":"Communications protocol","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.26600000262260437}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1002/cpe.70500","is_oa":true,"landing_page_url":"https://doi.org/10.1002/cpe.70500","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cpe.70500","source":{"id":"https://openalex.org/S11065456","display_name":"Concurrency and Computation Practice and Experience","issn_l":"1532-0626","issn":["1532-0626","1532-0634"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Concurrency and Computation: Practice and Experience","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1002/cpe.70500","is_oa":true,"landing_page_url":"https://doi.org/10.1002/cpe.70500","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cpe.70500","source":{"id":"https://openalex.org/S11065456","display_name":"Concurrency and Computation Practice and Experience","issn_l":"1532-0626","issn":["1532-0626","1532-0634"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Concurrency and Computation: Practice and Experience","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7123346683.pdf","grobid_xml":"https://content.openalex.org/works/W7123346683.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W2057332538","https://openalex.org/W2757498728","https://openalex.org/W2969388332","https://openalex.org/W2973435160","https://openalex.org/W4416146729"],"related_works":[],"abstract_inverted_index":{"ABSTRACT":[0],"As":[1],"model":[2],"parameters":[3],"increase":[4],"exponentially,":[5],"distributed":[6,20],"training":[7,21,28,54,75,300],"has":[8,46],"become":[9],"essential":[10],"for":[11,53,63,69,125,154,232],"advancing":[12],"modern":[13],"deep":[14],"neural":[15],"networks.":[16],"Megatron\u2010LM,":[17],"an":[18,264,282],"efficient":[19],"framework":[22],"developed":[23],"by":[24,36],"NVIDIA,":[25],"enables":[26],"the":[27,101,126,172,196,207,248,274,296],"of":[29,34,59,90,103,174,211,250,268,285,299],"trillion\u2010parameter":[30],"models":[31],"on":[32,96],"thousands":[33],"GPUs":[35],"integrating":[37],"tensor,":[38],"pipeline,":[39],"and":[40,87,112,128,165,189,254,291],"data":[41,289],"parallelism.":[42],"Its":[43],"computational":[44,71,127,233,240],"efficiency":[45],"established":[47],"it":[48],"as":[49,79,146],"a":[50,80,107,147,238],"foundational":[51],"tool":[52],"large\u2010scale":[55,104],"models.":[56],"Rapid":[57],"identification":[58],"optimal":[60],"parallel":[61,161,175,202,290,293],"configurations":[62],"specific":[64],"GPU":[65,134,156],"clusters":[66,106],"is":[67,144,244],"critical":[68],"maximizing":[70],"resource":[72,111],"utilization,":[73],"with":[74],"time":[76,301],"prediction":[77,117,150,266,302],"serving":[78],"key":[81],"evaluation":[82],"metric.":[83],"The":[84],"high":[85],"cost":[86,113],"limited":[88],"availability":[89],"high\u2010performance":[91],"GPUs,":[92,183,237],"particularly":[93],"those":[94],"based":[95],"NVIDIA":[97],"architectures,":[98],"have":[99],"made":[100],"construction":[102],"heterogeneous":[105,133,155,182,227,236,288,305],"practical":[108],"solution":[109],"to":[110,170,194,230,246,304],"constraints.":[114],"However,":[115],"existing":[116],"methods":[118],"do":[119],"not":[120],"reliably":[121],"or":[122],"efficiently":[123],"account":[124,231],"communication":[129,179,197,212],"complexities":[130],"inherent":[131],"in":[132,200,225,270,287],"clusters.":[135,157],"To":[136,177],"address":[137,178],"this":[138],"gap,":[139],"HATP":[140,163,262,279],"(Heterogeneous\u2010Aware":[141],"Time":[142],"Predictor)":[143],"introduced":[145],"novel":[148],"performance":[149,219,241],"method":[151],"specifically":[152],"designed":[153],"For":[158],"any":[159],"given":[160],"configuration,":[162],"rapidly":[164],"accurately":[166],"simulates":[167],"execution":[168],"times":[169],"inform":[171],"optimization":[173],"strategies.":[176,203],"differences":[180,234],"among":[181,235],"comprehensive":[184],"experimental":[185],"analyses":[186],"are":[187,192],"conducted":[188],"analytical":[190],"expressions":[191],"derived":[193],"characterize":[195],"frequency":[198],"patterns":[199],"Megatron\u2010LM's":[201],"This":[204],"work":[205],"presents":[206],"first":[208,297],"systematic":[209],"quantification":[210],"operations":[213],"within":[214],"Megatron\u2010LM":[215],"framework,":[216],"ensuring":[217],"that":[218,261],"predictions":[220],"remain":[221],"highly":[222],"accurate":[223],"even":[224],"complex,":[226],"environments.":[228,306],"Furthermore,":[229],"layer\u2010level":[239],"acquisition":[242],"scheme":[243],"proposed":[245],"reduce":[247],"impact":[249],"fine\u2010grained":[251],"operator":[252],"overlap":[253],"additional":[255],"memory":[256],"operations.":[257],"Experimental":[258],"results":[259],"demonstrate":[260],"achieves":[263],"average":[265,283],"accuracy":[267,284],"97.41%":[269],"isomorphic":[271],"environments,":[272],"surpassing":[273],"current":[275],"state\u2010of\u2010the\u2010art":[276],"method,":[277],"ACEso.":[278],"also":[280],"attains":[281],"96.04%":[286],"pipeline":[292],"configurations,":[294],"representing":[295],"extension":[298],"capabilities":[303]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2026-01-14T00:00:00"}