{"id":"https://openalex.org/W4405756071","doi":"https://doi.org/10.1109/sc41406.2024.00094","title":"Optimizing Distributed ML Communication with Fused Computation-Collective Operations","display_name":"Optimizing Distributed ML Communication with Fused Computation-Collective Operations","alternate_titles":[],"publication_year":2024,"publication_date":"2024-11-17","ids":{"openalex":"https://openalex.org/W4405756071","doi":"https://doi.org/10.1109/sc41406.2024.00094"},"language":"en","primary_location":{"id":"doi:10.1109/sc41406.2024.00094","doi":"https://doi.org/10.1109/sc41406.2024.00094","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sc41406.2024.00094","pdf_url":null,"source":{"id":"https://openalex.org/S4363608557","display_name":"International Conference for High Performance Computing, Networking, Storage and Analysis (SC)","issn_l":"2167-4337","issn":["2167-4337"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SC24: International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"parent_works":[{"id":"https://openalex.org/W4405755563","display_name":"SC24: International Conference for High Performance Computing, Networking, Storage and Analysis","relationship":"volume","link_type":"volume"}],"lineage":["https://openalex.org/W4405756071","https://openalex.org/W4405755563"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.06942","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049343138","display_name":"Kishore Punniyamurthy","orcid":"https://orcid.org/0009-0006-2311-8417","observed_orcids":["https://orcid.org/0009-0006-2311-8417"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Kishore Punniyamurthy","raw_affiliation_strings":["AMD Research and Advanced Development"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMD Research and Advanced Development","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048039700","display_name":"Khaled Hamidouche","orcid":"https://orcid.org/0000-0003-4836-5335","observed_orcids":["https://orcid.org/0000-0003-4836-5335"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Khaled Hamidouche","raw_affiliation_strings":["AMD Research and Advanced Development"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMD Research and Advanced Development","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077056823","display_name":"Bradford M. Beckmann","orcid":"https://orcid.org/0000-0002-5444-6521","observed_orcids":["https://orcid.org/0000-0002-5444-6521"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Bradford M. Beckmann","raw_affiliation_strings":["AMD Research and Advanced Development"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AMD Research and Advanced Development","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.7154,"has_fulltext":true,"cited_by_count":52,"reviewed_by_count":0,"citation_normalized_percentile":{"value":0.98691369,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"17"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/collective-communication","display_name":"collective communication","score":0.8889999985694885},{"id":"https://openalex.org/keywords/computation-communication-overlap","display_name":"communication-computation overlap","score":0.6679999828338623},{"id":"https://openalex.org/keywords/distributed-learning","display_name":"distributed learning","score":0.7009999752044678},{"id":"https://openalex.org/keywords/allreduce","display_name":"AllReduce","score":0.6359999775886536},{"id":"https://openalex.org/keywords/gemm","display_name":"GEMM","score":0.9879999756813049},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.7839999794960022},{"id":"https://openalex.org/keywords/moe","display_name":"MoE","score":0.8460000157356262},{"id":"https://openalex.org/keywords/triton","display_name":"Triton","score":0.8040000200271606},{"id":"https://openalex.org/keywords/work-teams","display_name":"work teams","score":0.9380000233650208}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7271541357040405},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6252437233924866},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5092900991439819},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.32245349884033203},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.12683773040771484}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/sc41406.2024.00094","doi":"https://doi.org/10.1109/sc41406.2024.00094","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sc41406.2024.00094","pdf_url":null,"source":{"id":"https://openalex.org/S4363608557","display_name":"International Conference for High Performance Computing, Networking, Storage and Analysis (SC)","issn_l":"2167-4337","issn":["2167-4337"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SC24: International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2305.06942","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.06942","pdf_url":"https://arxiv.org/pdf/2305.06942","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2305.06942","doi":"https://doi.org/10.48550/arxiv.2305.06942","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.06942","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.06942","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.06942","pdf_url":"https://arxiv.org/pdf/2305.06942","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4405756071.pdf","grobid_xml":"https://content.openalex.org/works/W4405756071.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W2029517391","https://openalex.org/W2134427337","https://openalex.org/W2581624817","https://openalex.org/W2584650020","https://openalex.org/W2605251767","https://openalex.org/W2732691924","https://openalex.org/W2947737663","https://openalex.org/W2954698171","https://openalex.org/W2973727699","https://openalex.org/W2992165038","https://openalex.org/W2998812396","https://openalex.org/W3039165326","https://openalex.org/W3072623287","https://openalex.org/W3096425133","https://openalex.org/W3119866685","https://openalex.org/W3133174490","https://openalex.org/W3190806564","https://openalex.org/W3193985311","https://openalex.org/W3197720002","https://openalex.org/W4220884018","https://openalex.org/W4226515448","https://openalex.org/W4281790033","https://openalex.org/W4281922990","https://openalex.org/W4284974526","https://openalex.org/W4312060029","https://openalex.org/W4384775699","https://openalex.org/W4392265896","https://openalex.org/W4394998995"],"related_works":["https://openalex.org/W7163987664","https://openalex.org/W7213348762","https://openalex.org/W7156599693","https://openalex.org/W4399694321","https://openalex.org/W7222266945","https://openalex.org/W7214721677","https://openalex.org/W4417295212","https://openalex.org/W3193985311","https://openalex.org/W4413915532","https://openalex.org/W7220963610"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"models":[2],"are":[3],"distributed":[4],"across":[5,151],"multiple":[6],"nodes":[7],"using":[8],"numerous":[9],"parallelism":[10],"strategies.":[11],"The":[12],"resulting":[13],"collective":[14,44],"communication":[15,45,57,96,131],"is":[16],"often":[17],"on":[18],"the":[19,78,95,117],"critical":[20],"path":[21],"due":[22],"to":[23,32,55,67,93,120],"a":[24],"lack":[25],"of":[26],"independent":[27],"coarse-grain":[28],"computation":[29,40],"kernels":[30,108],"available":[31],"execute.":[33],"In":[34],"this":[35],"work,":[36],"we":[37],"propose":[38],"fusing":[39],"with":[41,52,132],"its":[42],"subsequent":[43],"and":[46,58,91,101],"leverage":[47],"GPUs\u2019":[48],"massive":[49],"parallelism,":[50],"along":[51],"GPU-initiated":[53],"communication,":[54],"overlap":[56],"computation.":[59,82],"Specifically":[60],"threadblocks/workgroups":[61],"(WGs)":[62],"immediately":[63],"communicate":[64],"their":[65,72,122,136],"results":[66],"remote":[68],"GPUs":[69],"after":[70],"completing":[71],"computation,":[73],"while":[74],"other":[75],"WGs":[76],"within":[77],"same":[79],"kernel":[80],"perform":[81],"We":[83,105],"developed":[84],"three":[85,153],"prototype":[86],"fused":[87,107],"operators":[88],"(embedding+All-toAll,":[89],"GEMV+AllReduce,":[90],"GEMM+All-to-All)":[92],"address":[94],"overheads":[97],"in":[98],"DLRM,":[99],"Transformers":[100],"MoE":[102],"model":[103],"architectures.":[104],"expose":[106],"as":[109,113,115],"new":[110],"PyTorch":[111],"operators,":[112],"well":[114],"extend":[116],"Triton":[118],"framework":[119],"demonstrate":[121],"practicality.":[123],"Our":[124],"evaluations":[125],"show":[126],"our":[127],"approach":[128],"effectively":[129],"overlaps":[130],"computations,":[133],"subsequently":[134],"reducing":[135],"combined":[137],"execution":[138,149],"time":[139,150],"achieving":[140],"$\\mathbf{1":[141],"2":[142],"\\%}$":[143,147],"-":[144],"$\\mathbf{3":[145],"1":[146],"lower":[148],"all":[152],"operators.":[154]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4405756071","counts_by_year":[{"year":2026,"cited_by_count":28},{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":7}],"updated_date":"2026-11-22T12:46:00.618207","created_date":"2024-12-25T00:00:00"}
