{"id":"https://openalex.org/W4391341581","doi":"https://doi.org/10.1109/mm.2024.3360081","title":"High-Speed Data Communication With Advanced Networks in Large Language Model Training","display_name":"High-Speed Data Communication With Advanced Networks in Large Language Model Training","publication_year":2024,"publication_date":"2024-01-30","ids":{"openalex":"https://openalex.org/W4391341581","doi":"https://doi.org/10.1109/mm.2024.3360081"},"language":"en","primary_location":{"id":"doi:10.1109/mm.2024.3360081","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mm.2024.3360081","pdf_url":null,"source":{"id":"https://openalex.org/S59697426","display_name":"IEEE Micro","issn_l":"0272-1732","issn":["0272-1732","1937-4143"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Micro","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074001551","display_name":"Liuyao Dai","orcid":"https://orcid.org/0000-0002-0907-6920"},"institutions":[{"id":"https://openalex.org/I156087764","display_name":"University of California, Merced","ror":"https://ror.org/00d9ah105","country_code":"US","type":"education","lineage":["https://openalex.org/I156087764"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Liuyao Dai","raw_affiliation_strings":["University of California Merced, Merced, CA, 95343, USA"],"affiliations":[{"raw_affiliation_string":"University of California Merced, Merced, CA, 95343, USA","institution_ids":["https://openalex.org/I156087764"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091038854","display_name":"Hao Qi","orcid":"https://orcid.org/0009-0007-8795-5262"},"institutions":[{"id":"https://openalex.org/I156087764","display_name":"University of California, Merced","ror":"https://ror.org/00d9ah105","country_code":"US","type":"education","lineage":["https://openalex.org/I156087764"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hao Qi","raw_affiliation_strings":["University of California Merced, Merced, CA, 95343, USA"],"affiliations":[{"raw_affiliation_string":"University of California Merced, Merced, CA, 95343, USA","institution_ids":["https://openalex.org/I156087764"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101826240","display_name":"Weicong Chen","orcid":"https://orcid.org/0000-0003-0573-8808"},"institutions":[{"id":"https://openalex.org/I156087764","display_name":"University of California, Merced","ror":"https://ror.org/00d9ah105","country_code":"US","type":"education","lineage":["https://openalex.org/I156087764"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Weicong Chen","raw_affiliation_strings":["University of California Merced, Merced, CA, 95343, USA"],"affiliations":[{"raw_affiliation_string":"University of California Merced, Merced, CA, 95343, USA","institution_ids":["https://openalex.org/I156087764"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067748041","display_name":"Xiaoyi Lu","orcid":"https://orcid.org/0000-0001-7581-8905"},"institutions":[{"id":"https://openalex.org/I156087764","display_name":"University of California, Merced","ror":"https://ror.org/00d9ah105","country_code":"US","type":"education","lineage":["https://openalex.org/I156087764"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoyi Lu","raw_affiliation_strings":["University of California Merced, Merced, CA, 95343, USA"],"affiliations":[{"raw_affiliation_string":"University of California Merced, Merced, CA, 95343, USA","institution_ids":["https://openalex.org/I156087764"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5074001551"],"corresponding_institution_ids":["https://openalex.org/I156087764"],"apc_list":null,"apc_paid":null,"fwci":4.0799,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.94253346,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"44","issue":"2","first_page":"31","last_page":"40"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13062","display_name":"Cognitive Computing and Networks","score":0.7702000141143799,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13062","display_name":"Cognitive Computing and Networks","score":0.7702000141143799,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14470","display_name":"Advanced Data Processing Techniques","score":0.6711000204086304,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.6528000235557556,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8953714370727539},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6081275939941406},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5292888283729553},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.45709317922592163},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.44403403997421265},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.42595940828323364},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.347156822681427},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.3366389572620392},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3267174959182739},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.20228558778762817}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8953714370727539},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6081275939941406},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5292888283729553},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.45709317922592163},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.44403403997421265},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.42595940828323364},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.347156822681427},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3366389572620392},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3267174959182739},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.20228558778762817},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mm.2024.3360081","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mm.2024.3360081","pdf_url":null,"source":{"id":"https://openalex.org/S59697426","display_name":"IEEE Micro","issn_l":"0272-1732","issn":["0272-1732","1937-4143"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Micro","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G551163407","display_name":null,"funder_award_id":"2019144","funder_id":"https://openalex.org/F4320333199","funder_display_name":"Materials Research Institute, Pennsylvania State University"},{"id":"https://openalex.org/G6938478198","display_name":null,"funder_award_id":"2340982","funder_id":"https://openalex.org/F4320337377","funder_display_name":"Office of Advanced Cyberinfrastructure"},{"id":"https://openalex.org/G793108612","display_name":null,"funder_award_id":"2321123","funder_id":"https://openalex.org/F4320337377","funder_display_name":"Office of Advanced Cyberinfrastructure"}],"funders":[{"id":"https://openalex.org/F4320333199","display_name":"Materials Research Institute, Pennsylvania State University","ror":null},{"id":"https://openalex.org/F4320337377","display_name":"Office of Advanced Cyberinfrastructure","ror":"https://ror.org/04nh1dc89"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W2168231600","https://openalex.org/W2174359540","https://openalex.org/W2198867094","https://openalex.org/W2298436731","https://openalex.org/W2618530766","https://openalex.org/W2896457183","https://openalex.org/W2991040477","https://openalex.org/W3204998121","https://openalex.org/W4288089799","https://openalex.org/W4387828615","https://openalex.org/W6684859321","https://openalex.org/W6755207826","https://openalex.org/W6769627184"],"related_works":["https://openalex.org/W2317245370","https://openalex.org/W4249323025","https://openalex.org/W198851386","https://openalex.org/W1980160788","https://openalex.org/W2030310580","https://openalex.org/W947442053","https://openalex.org/W2148915962","https://openalex.org/W4313433561","https://openalex.org/W2283866686","https://openalex.org/W2948365806"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"like":[4],"GPT,":[5],"BERT,":[6],"and":[7,32,37,41,47,54,67,87,112,115,144],"T5":[8],"are":[9],"pivotal":[10],"in":[11,56,60,102,118,140],"natural":[12],"language":[13],"processing.":[14],"Their":[15],"distributed":[16,141],"training":[17,27,110,143],"is":[18],"influenced":[19],"by":[20],"high-speed":[21],"interconnects.":[22],"This":[23],"study":[24],"characterizes":[25],"their":[26],"performance":[28],"across":[29],"various":[30],"interconnects":[31],"communication":[33,98,130,139],"protocols:":[34],"TCP/IP,":[35],"IPoIB,":[36],"RDMA,":[38],"using":[39],"data":[40,57,103],"model":[42,61,119],"parallelism.":[43],"RDMA-100Gbps":[44],"outperforms":[45],"IPoIB-100Gbps":[46],"TCP/IP-10Gbps,":[48],"with":[49,82,89,100],"average":[50],"gains":[51,64],"of":[52,109,138],"2.5x":[53],"4.8x":[55],"parallelism,":[58,62],"while":[59],"the":[63,71,136],"were":[65],"1.1x":[66],"1.2x.":[68],"RDMA":[69],"achieves":[70],"highest":[72],"interconnect":[73],"utilization":[74],"(up":[75],"to":[76,80,84,91,107,123],"60":[77],"Gbps),":[78],"compared":[79],"IPoIB":[81],"up":[83,90,106,122],"20":[85],"Gbps":[86],"TCP/IP":[88],"9":[92],"Gbps.":[93],"Larger":[94],"models":[95],"demand":[96],"increased":[97],"bandwidth,":[99],"AllReduce":[101,117],"parallelism":[104,120],"consuming":[105],"91%":[108],"time,":[111],"forward":[113],"receive":[114],"Back-Embedding":[116],"taking":[121],"90%.":[124],"The":[125],"larger-scale":[126],"experiment":[127],"confirms":[128],"that":[129],"predominates":[131],"iterations.":[132],"Our":[133],"findings":[134],"underscore":[135],"significance":[137],"LLM":[142],"present":[145],"opportunities":[146],"for":[147],"optimization.":[148]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":3}],"updated_date":"2026-04-11T08:14:18.477133","created_date":"2025-10-10T00:00:00"}