{"id":"https://openalex.org/W2606891064","doi":"https://doi.org/10.18653/v1/d17-1045","title":"Sparse Communication for Distributed Gradient Descent","display_name":"Sparse Communication for Distributed Gradient Descent","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2606891064","doi":"https://doi.org/10.18653/v1/d17-1045","mag":"2606891064"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d17-1045","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1045","pdf_url":"https://www.aclweb.org/anthology/D17-1045.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D17-1045.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Alham Fikri Aji","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Alham Fikri Aji","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Kenneth Heafield","orcid":null},"institutions":[{"id":"https://openalex.org/I1320481043","display_name":"European Union","ror":"https://ror.org/019w4f821","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043"]},{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["BE","GB"],"is_corresponding":false,"raw_author_name":"Kenneth Heafield","raw_affiliation_strings":["School of Informatics, University of Edinburgh 10 Crichton Street Edinburgh EH8 9AB Scotland, European Union"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Informatics, University of Edinburgh 10 Crichton Street Edinburgh EH8 9AB Scotland, European Union","institution_ids":["https://openalex.org/I1320481043","https://openalex.org/I98677209"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.5776,"has_fulltext":true,"cited_by_count":500,"citation_normalized_percentile":{"value":0.98252744,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"440","last_page":"445"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mnist-database","display_name":"MNIST database","score":0.9297999739646912},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.7419000267982483},{"id":"https://openalex.org/keywords/stochastic-gradient-descent","display_name":"Stochastic gradient descent","score":0.5716000199317932},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5462999939918518},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.492000013589859},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.44999998807907104},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.44769999384880066},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.38769999146461487}],"concepts":[{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.9297999739646912},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.7419000267982483},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5742999911308289},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.5716000199317932},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5462999939918518},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4943999946117401},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.492000013589859},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.44999998807907104},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.44769999384880066},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42739999294281006},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.38769999146461487},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.37059998512268066},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.35420000553131104},{"id":"https://openalex.org/C115680565","wikidata":"https://www.wikidata.org/wiki/Q5977448","display_name":"Gradient method","level":2,"score":0.3479999899864197},{"id":"https://openalex.org/C2776637919","wikidata":"https://www.wikidata.org/wiki/Q624380","display_name":"Descent (aeronautics)","level":2,"score":0.33329999446868896},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.3050000071525574},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.2879999876022339},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.2702000141143799},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.26809999346733093},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.2653000056743622},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.2612000107765198}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/d17-1045","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1045","pdf_url":"https://www.aclweb.org/anthology/D17-1045.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1704.05021","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1704.05021","pdf_url":"https://arxiv.org/pdf/1704.05021","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:pure.ed.ac.uk:publications/242734fb-0f93-4ce8-9c88-8dba67f70fee","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/242734fb-0f93-4ce8-9c88-8dba67f70fee","pdf_url":"https://www.research.ed.ac.uk/files/40918571/drop_paper.pdf","source":{"id":"https://openalex.org/S4406922455","display_name":"Edinburgh Research Explorer","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Aji, A & Heafield, K 2017, Sparse Communication for Distributed Gradient Descent. in Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing (EMNLP 2017). Copenhagen, Denmark , pp. 440-445, EMNLP 2017: Conference on Empirical Methods in Natural Language Processing, Copenhagen, Denmark, 7/09/17. https://doi.org/10.18653/v1/D17-1045","raw_type":"contributionToPeriodical"},{"id":"pmh:oai:pure.ed.ac.uk:openaire/242734fb-0f93-4ce8-9c88-8dba67f70fee","is_oa":true,"landing_page_url":"https://hdl.handle.net/20.500.11820/242734fb-0f93-4ce8-9c88-8dba67f70fee","pdf_url":null,"source":{"id":"https://openalex.org/S4406922455","display_name":"Edinburgh Research Explorer","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Aji, A & Heafield, K 2017, Sparse Communication for Distributed Gradient Descent. in Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing (EMNLP 2017). Copenhagen, Denmark , pp. 440-445, EMNLP 2017: Conference on Empirical Methods in Natural Language Processing, Copenhagen, Denmark, 7/09/17. https://doi.org/10.18653/v1/D17-1045","raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.18653/v1/d17-1045","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1045","pdf_url":"https://www.aclweb.org/anthology/D17-1045.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"},{"id":"https://openalex.org/F4320313467","display_name":"Alan Turing Institute","ror":"https://ror.org/035dkdb55"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2606891064.pdf","grobid_xml":"https://content.openalex.org/works/W2606891064.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,54],"make":[1],"distributed":[2],"stochastic":[3],"gradient":[4],"descent":[5],"faster":[6],"by":[7],"exchanging":[8],"sparse":[9,40],"updates":[10,16,22,32],"instead":[11],"of":[12],"dense":[13],"updates.":[14],"Gradient":[15],"are":[17,23],"positively":[18],"skewed":[19],"as":[20],"most":[21],"near":[24],"zero,":[25],"so":[26],"we":[27,91],"map":[28],"the":[29,52,82,107],"99%":[30],"smallest":[31],"(by":[33],"absolute":[34],"value)":[35],"to":[36,49,61,95],"zero":[37],"then":[38],"exchange":[39],"matrices.":[41],"This":[42],"method":[43],"can":[44,92],"be":[45],"combined":[46],"with":[47],"quantization":[48],"further":[50],"improve":[51],"compression.":[53],"explore":[55],"different":[56,76],"configurations":[57,71,77],"and":[58,65,101],"apply":[59],"them":[60],"neural":[62],"machine":[63],"translation":[64,85],"MNIST":[66,100],"image":[67],"classification":[68],"tasks.":[69],"Most":[70],"work":[72],"on":[73,81,99,103],"MNIST,":[74],"whereas":[75],"reduce":[78],"convergence":[79],"rate":[80],"more":[83],"complex":[84],"task.":[86],"Our":[87],"experiments":[88],"show":[89],"that":[90],"achieve":[93],"up":[94,98],"49%":[96],"speed":[97],"22%":[102],"NMT":[104],"without":[105],"damaging":[106],"final":[108],"accuracy":[109],"or":[110],"BLEU.":[111]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":77},{"year":2024,"cited_by_count":98},{"year":2023,"cited_by_count":78},{"year":2022,"cited_by_count":74},{"year":2021,"cited_by_count":81},{"year":2020,"cited_by_count":53},{"year":2019,"cited_by_count":22},{"year":2018,"cited_by_count":7}],"updated_date":"2026-05-12T08:28:47.272897","created_date":"2017-04-28T00:00:00"}