{"id":"https://openalex.org/W4404385992","doi":"https://doi.org/10.1145/3698038.3698541","title":"Near-Lossless Gradient Compression for Data-Parallel Distributed DNN Training","display_name":"Near-Lossless Gradient Compression for Data-Parallel Distributed DNN Training","publication_year":2024,"publication_date":"2024-11-14","ids":{"openalex":"https://openalex.org/W4404385992","doi":"https://doi.org/10.1145/3698038.3698541"},"language":"en","primary_location":{"id":"doi:10.1145/3698038.3698541","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3698038.3698541","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Symposium on Cloud Computing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100677771","display_name":"Xue Li","orcid":"https://orcid.org/0000-0001-5713-7225"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Xue Li","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113292370","display_name":"Cheng Guo","orcid":"https://orcid.org/0000-0001-6785-8288"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cheng Guo","raw_affiliation_strings":["Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008523628","display_name":"Kun Qian","orcid":"https://orcid.org/0000-0001-9882-9279"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kun Qian","raw_affiliation_strings":["Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031176052","display_name":"Menghao Zhang","orcid":"https://orcid.org/0000-0001-5274-5512"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Menghao Zhang","raw_affiliation_strings":["Unaffiliated"],"affiliations":[{"raw_affiliation_string":"Unaffiliated","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101690665","display_name":"Mengyu Yang","orcid":"https://orcid.org/0000-0001-7832-0926"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mengyu Yang","raw_affiliation_strings":["Unaffiliated"],"affiliations":[{"raw_affiliation_string":"Unaffiliated","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100771111","display_name":"Mingwei Xu","orcid":"https://orcid.org/0000-0002-4847-4585"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingwei Xu","raw_affiliation_strings":["Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100677771"],"corresponding_institution_ids":["https://openalex.org/I4210095624"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.22712527,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"977","last_page":"994"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12702","display_name":"Brain Tumor Detection and Classification","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/2808","display_name":"Neurology"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T12702","display_name":"Brain Tumor Detection and Classification","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/2808","display_name":"Neurology"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lossless-compression","display_name":"Lossless compression","score":0.8389186859130859},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7478979825973511},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.621076226234436},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5590561628341675},{"id":"https://openalex.org/keywords/lossy-compression","display_name":"Lossy compression","score":0.5421231389045715},{"id":"https://openalex.org/keywords/compression","display_name":"Compression (physics)","score":0.536470353603363},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35981833934783936}],"concepts":[{"id":"https://openalex.org/C81081738","wikidata":"https://www.wikidata.org/wiki/Q55542","display_name":"Lossless compression","level":3,"score":0.8389186859130859},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7478979825973511},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.621076226234436},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5590561628341675},{"id":"https://openalex.org/C165021410","wikidata":"https://www.wikidata.org/wiki/Q55564","display_name":"Lossy compression","level":2,"score":0.5421231389045715},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.536470353603363},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35981833934783936},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3698038.3698541","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3698038.3698541","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Symposium on Cloud Computing","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/13","score":0.550000011920929,"display_name":"Climate action"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W105472389","https://openalex.org/W1583837637","https://openalex.org/W2060108852","https://openalex.org/W2107745473","https://openalex.org/W2117539524","https://openalex.org/W2142265837","https://openalex.org/W2194775991","https://openalex.org/W2407022425","https://openalex.org/W2767510344","https://openalex.org/W2786602455","https://openalex.org/W3072623287","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3129488589","https://openalex.org/W3175449831","https://openalex.org/W3206636350","https://openalex.org/W4205983429","https://openalex.org/W4221008227","https://openalex.org/W4230791431","https://openalex.org/W4230874317","https://openalex.org/W4233996382","https://openalex.org/W4254620533","https://openalex.org/W4290991009","https://openalex.org/W4312429406","https://openalex.org/W4318541593","https://openalex.org/W4372267133","https://openalex.org/W4375928371","https://openalex.org/W4389628091"],"related_works":["https://openalex.org/W2385628723","https://openalex.org/W2547124190","https://openalex.org/W3180760233","https://openalex.org/W3035703949","https://openalex.org/W4247601675","https://openalex.org/W1970394887","https://openalex.org/W755971114","https://openalex.org/W2118338613","https://openalex.org/W1982468865","https://openalex.org/W4210455546"],"abstract_inverted_index":{"Data":[0],"parallelism":[1],"has":[2,27],"become":[3],"a":[4,30,40,60,99],"cornerstone":[5],"in":[6,42],"scaling":[7],"up":[8,151],"the":[9,17,52,68,111,121,138],"training":[10,35,46,73,124,148],"of":[11,55,113,130],"deep":[12],"neural":[13],"networks":[14],"(DNNs).":[15],"However,":[16],"communication":[18,69,139],"overhead":[19],"associated":[20],"with":[21],"synchronizing":[22],"gradients":[23],"across":[24],"multiple":[25],"nodes":[26],"emerged":[28],"as":[29,142,144],"significant":[31],"bottleneck,":[32],"adversely":[33],"affecting":[34,120],"efficiency":[36],"and":[37,146],"leading":[38],"to":[39,66,82,152],"surge":[41],"large-scale":[43],"distributed":[44],"model":[45],"costs.":[47],"By":[48],"leveraging":[49],"insights":[50],"into":[51],"statistical":[53],"characteristics":[54],"gradients,":[56],"we":[57],"present":[58],"GComp,":[59,131],"near-lossless":[61],"gradient":[62,84],"compression":[63],"scheme":[64],"designed":[65],"reduce":[67,110],"burden":[70],"during":[71],"data-parallel":[72],"significantly.":[74],"GComp":[75],"develops":[76],"an":[77,90],"optimized":[78],"Huffman":[79],"encoding/decoding":[80],"strategy":[81,101],"compress":[83],"exponents":[85],"effectively.":[86],"Additionally,":[87],"it":[88],"introduces":[89],"innovative":[91],"multi-level":[92],"quantization":[93],"method":[94,135],"for":[95,115],"mantissa,":[96],"complemented":[97],"by":[98,141,150],"pruning":[100],"that":[102,133],"eliminates":[103],"zero-valued":[104],"gradients.":[105],"These":[106],"integrated":[107],"approaches":[108],"significantly":[109],"volume":[112,140],"data":[114],"synchronization,":[116],"while":[117],"virtually":[118],"not":[119],"DNN":[122],"model's":[123],"accuracy.":[125],"We":[126],"conduct":[127],"comprehensive":[128],"evaluations":[129],"demonstrating":[132],"our":[134],"can":[136],"decrease":[137],"much":[143],"67.1%,":[145],"enhance":[147],"speed":[149],"1.9\u00d7.":[153]},"counts_by_year":[],"updated_date":"2025-12-19T19:40:27.379048","created_date":"2025-10-10T00:00:00"}