{"id":"https://openalex.org/W4286374952","doi":"https://doi.org/10.1109/jsac.2022.3192050","title":"AC-SGD: Adaptively Compressed SGD for Communication-Efficient Distributed Learning","display_name":"AC-SGD: Adaptively Compressed SGD for Communication-Efficient Distributed Learning","publication_year":2022,"publication_date":"2022-07-20","ids":{"openalex":"https://openalex.org/W4286374952","doi":"https://doi.org/10.1109/jsac.2022.3192050"},"language":"en","primary_location":{"id":"doi:10.1109/jsac.2022.3192050","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsac.2022.3192050","pdf_url":null,"source":{"id":"https://openalex.org/S90422530","display_name":"IEEE Journal on Selected Areas in Communications","issn_l":"0733-8716","issn":["0733-8716","1558-0008"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Communications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039893040","display_name":"Guangfeng Yan","orcid":"https://orcid.org/0000-0002-0936-9467"},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]},{"id":"https://openalex.org/I4210105229","display_name":"City University of Hong Kong, Shenzhen Research Institute","ror":"https://ror.org/00xc0ma20","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210105229"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Guangfeng Yan","raw_affiliation_strings":["Department of Computer Science, City University of Hong Kong, Kowloon Tong, Hong Kong","City University of Hong Kong Shenzhen Research Institute, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-0936-9467","affiliations":[{"raw_affiliation_string":"Department of Computer Science, City University of Hong Kong, Kowloon Tong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]},{"raw_affiliation_string":"City University of Hong Kong Shenzhen Research Institute, Shenzhen, China","institution_ids":["https://openalex.org/I168719708","https://openalex.org/I4210105229"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100428606","display_name":"Tan Li","orcid":"https://orcid.org/0000-0001-6129-4792"},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]},{"id":"https://openalex.org/I4210105229","display_name":"City University of Hong Kong, Shenzhen Research Institute","ror":"https://ror.org/00xc0ma20","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210105229"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Tan Li","raw_affiliation_strings":["Department of Computer Science, City University of Hong Kong, Kowloon Tong, Hong Kong","City University of Hong Kong Shenzhen Research Institute, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0001-6129-4792","affiliations":[{"raw_affiliation_string":"Department of Computer Science, City University of Hong Kong, Kowloon Tong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]},{"raw_affiliation_string":"City University of Hong Kong Shenzhen Research Institute, Shenzhen, China","institution_ids":["https://openalex.org/I168719708","https://openalex.org/I4210105229"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088293566","display_name":"Shao\u2010Lun Huang","orcid":"https://orcid.org/0000-0003-2827-4022"},"institutions":[{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shao-Lun Huang","raw_affiliation_strings":["Data Science and Information Technology Research Center, Tsinghua-Berkeley Shenzhen Institute, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-2827-4022","affiliations":[{"raw_affiliation_string":"Data Science and Information Technology Research Center, Tsinghua-Berkeley Shenzhen Institute, Shenzhen, China","institution_ids":["https://openalex.org/I4210114105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018464968","display_name":"Tian Lan","orcid":"https://orcid.org/0000-0003-3010-8090"},"institutions":[{"id":"https://openalex.org/I193531525","display_name":"George Washington University","ror":"https://ror.org/00y4zzh67","country_code":"US","type":"education","lineage":["https://openalex.org/I193531525"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tian Lan","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The George Washington University, Washington, DC, USA"],"raw_orcid":"https://orcid.org/0000-0003-3010-8090","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The George Washington University, Washington, DC, USA","institution_ids":["https://openalex.org/I193531525"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035185924","display_name":"Linqi Song","orcid":"https://orcid.org/0000-0003-2756-4984"},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]},{"id":"https://openalex.org/I4210105229","display_name":"City University of Hong Kong, Shenzhen Research Institute","ror":"https://ror.org/00xc0ma20","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210105229"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Linqi Song","raw_affiliation_strings":["Department of Computer Science, City University of Hong Kong, Kowloon Tong, Hong Kong","City University of Hong Kong Shenzhen Research Institute, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-2756-4984","affiliations":[{"raw_affiliation_string":"Department of Computer Science, City University of Hong Kong, Kowloon Tong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]},{"raw_affiliation_string":"City University of Hong Kong Shenzhen Research Institute, Shenzhen, China","institution_ids":["https://openalex.org/I168719708","https://openalex.org/I4210105229"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.5553,"has_fulltext":false,"cited_by_count":37,"citation_normalized_percentile":{"value":0.97679372,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"40","issue":"9","first_page":"2678","last_page":"2693"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9914000034332275,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.868294358253479},{"id":"https://openalex.org/keywords/distributed-learning","display_name":"Distributed learning","score":0.4203079640865326},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3836478590965271},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3456149697303772},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.3434098958969116}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.868294358253479},{"id":"https://openalex.org/C2779582901","wikidata":"https://www.wikidata.org/wiki/Q21013010","display_name":"Distributed learning","level":2,"score":0.4203079640865326},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3836478590965271},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3456149697303772},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3434098958969116},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jsac.2022.3192050","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsac.2022.3192050","pdf_url":null,"source":{"id":"https://openalex.org/S90422530","display_name":"IEEE Journal on Selected Areas in Communications","issn_l":"0733-8716","issn":["0733-8716","1558-0008"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Communications","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4399999976158142,"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals"}],"awards":[{"id":"https://openalex.org/G1403791757","display_name":null,"funder_award_id":"ECS 21212419","funder_id":"https://openalex.org/F4320321592","funder_display_name":"Research Grants Council, University Grants Committee"},{"id":"https://openalex.org/G2015485856","display_name":null,"funder_award_id":"KQTD20170810150821146","funder_id":"https://openalex.org/F4320336569","funder_display_name":"Shenzhen Science and Technology Innovation Program"},{"id":"https://openalex.org/G2130087010","display_name":null,"funder_award_id":"7005660","funder_id":"https://openalex.org/F4320309893","funder_display_name":"City University of Hong Kong"},{"id":"https://openalex.org/G3043234225","display_name":null,"funder_award_id":"2021YFA0715202","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G7808265206","display_name":null,"funder_award_id":"7005561","funder_id":"https://openalex.org/F4320309893","funder_display_name":"City University of Hong Kong"},{"id":"https://openalex.org/G8326899987","display_name":null,"funder_award_id":"JSGG20201102162000001","funder_id":"https://openalex.org/F4320326705","funder_display_name":"Science, Technology and Innovation Commission of Shenzhen Municipality"}],"funders":[{"id":"https://openalex.org/F4320309893","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23"},{"id":"https://openalex.org/F4320316083","display_name":"Tencent","ror":"https://ror.org/00hhjss72"},{"id":"https://openalex.org/F4320321592","display_name":"Research Grants Council, University Grants Committee","ror":"https://ror.org/00djwmt25"},{"id":"https://openalex.org/F4320326705","display_name":"Science, Technology and Innovation Commission of Shenzhen Municipality","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null},{"id":"https://openalex.org/F4320336569","display_name":"Shenzhen Science and Technology Innovation Program","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W866281365","https://openalex.org/W1561921554","https://openalex.org/W1751687266","https://openalex.org/W2116612304","https://openalex.org/W2134130436","https://openalex.org/W2168231600","https://openalex.org/W2170240176","https://openalex.org/W2194775991","https://openalex.org/W2407022425","https://openalex.org/W2563351168","https://openalex.org/W2617766261","https://openalex.org/W2766140019","https://openalex.org/W2769644379","https://openalex.org/W2774000609","https://openalex.org/W2775776326","https://openalex.org/W2946815927","https://openalex.org/W2946900926","https://openalex.org/W2950346378","https://openalex.org/W2962705013","https://openalex.org/W2963179579","https://openalex.org/W2963433607","https://openalex.org/W2964163156","https://openalex.org/W3015640161","https://openalex.org/W3016081801","https://openalex.org/W3037047862","https://openalex.org/W3093502740","https://openalex.org/W3099852766","https://openalex.org/W3103657382","https://openalex.org/W3118608800","https://openalex.org/W3137328576","https://openalex.org/W3154560182","https://openalex.org/W3155611867","https://openalex.org/W3191321490","https://openalex.org/W3196370796","https://openalex.org/W4214711850","https://openalex.org/W4250589301","https://openalex.org/W4287327126","https://openalex.org/W4287863844","https://openalex.org/W4318619660","https://openalex.org/W6680008295","https://openalex.org/W6684859321","https://openalex.org/W6685053522","https://openalex.org/W6728757088","https://openalex.org/W6728975057","https://openalex.org/W6731031554","https://openalex.org/W6738460352","https://openalex.org/W6745723224","https://openalex.org/W6746200960","https://openalex.org/W6746839373","https://openalex.org/W6751646691","https://openalex.org/W6752191696","https://openalex.org/W6754416507","https://openalex.org/W6755251367","https://openalex.org/W6763138067","https://openalex.org/W6763393679","https://openalex.org/W6774827312","https://openalex.org/W6781049309","https://openalex.org/W6784072064","https://openalex.org/W6786422851","https://openalex.org/W6787972765","https://openalex.org/W6792231754","https://openalex.org/W6840443151"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Gradient":[0,80],"compression":[1,25,60,129,171,207,214],"(e.g.,":[2],"gradient":[3,6,24,59,213],"quantization":[4,64,89],"and":[5,48,65,91,105,137,190,199],"sparsification)":[7],"is":[8,26,68],"a":[9,29,45,56,76],"core":[10],"technique":[11],"in":[12,16,119,216],"reducing":[13],"communication":[14,103,134,180,220],"costs":[15],"distributed":[17],"learning":[18],"systems.":[19],"The":[20],"recent":[21],"trend":[22],"of":[23,32,53,88,100,109,122,218],"to":[27,84,97],"use":[28],"varying":[30],"number":[31,87,108],"bits":[33,90],"across":[34],"iterations,":[35],"however,":[36],"relying":[37],"on":[38,187,195],"empirical":[39],"observations":[40],"or":[41],"engineering":[42],"heuristics":[43],"without":[44],"systematic":[46],"treatment":[47],"analysis.":[49],"To":[50],"the":[51,86,92,98,102,106,123,145,155,165,211],"best":[52],"our":[54,206],"knowledge,":[55],"general":[57],"dynamic":[58,128],"that":[61,173,205],"leverages":[62],"both":[63],"sparsification":[66,93],"techniques":[67],"still":[69],"far":[70],"from":[71],"understanding.":[72],"This":[73],"paper":[74],"proposes":[75],"novel":[77],"Adaptively-Compressed":[78],"Stochastic":[79],"Descent":[81],"(AC-SGD)":[82],"strategy":[83],"adjust":[85],"size":[94],"with":[95],"respect":[96],"norm":[99],"gradients,":[101],"budget,":[104],"remaining":[107],"iterations.":[110],"In":[111],"particular,":[112],"we":[113,132,167,203],"derive":[114],"an":[115,139,169],"upper":[116],"bound,":[117],"tight":[118],"some":[120],"cases,":[121],"convergence":[124,158],"error":[125,159],"for":[126,153],"arbitrary":[127],"strategy.":[130],"Then":[131],"consider":[133],"budget":[135,181],"constraints":[136],"propose":[138],"optimization":[140],"formulation":[141],"-":[142,152],"denoted":[143],"as":[144],"<italic":[146],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[147],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Adaptive":[148],"Compression":[149],"Problem":[150],"(ACP)</i>":[151],"minimizing":[154],"deep":[156],"model\u2019s":[157],"under":[160,178],"such":[161],"constraints.":[162,182],"By":[163],"solving":[164],"ACP,":[166],"obtain":[168],"enhanced":[170],"algorithm":[172],"significantly":[174,209],"improves":[175],"model":[176],"accuracy":[177],"given":[179],"Finally,":[183],"through":[184],"extensive":[185],"experiments":[186],"computer":[188],"vision":[189],"natural":[191],"language":[192],"processing":[193],"tasks":[194],"MNIST,":[196],"CIFAR-10,":[197],"CIFAR-100":[198],"AG-News":[200],"datasets,":[201],"respectively,":[202],"demonstrate":[204],"scheme":[208],"outperforms":[210],"state-of-the-art":[212],"methods":[215],"terms":[217],"mitigating":[219],"costs.":[221]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}