{"id":"https://openalex.org/W4221157729","doi":"https://doi.org/10.48550/arxiv.2203.10991","title":"Minimum Variance Unbiased N:M Sparsity for the Neural Gradients","display_name":"Minimum Variance Unbiased N:M Sparsity for the Neural Gradients","publication_year":2022,"publication_date":"2022-03-21","ids":{"openalex":"https://openalex.org/W4221157729","doi":"https://doi.org/10.48550/arxiv.2203.10991"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2203.10991","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2203.10991","pdf_url":"https://arxiv.org/pdf/2203.10991","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2203.10991","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017964819","display_name":"Brian Chmiel","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Chmiel, Brian","raw_affiliation_strings":["Department of Electrical Engineering -Technion , Haifa , Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering -Technion , Haifa , Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087518651","display_name":"Itay Hubara","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Hubara, Itay","raw_affiliation_strings":["Department of Electrical Engineering -Technion , Haifa , Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering -Technion , Haifa , Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091675091","display_name":"Ron Banner","orcid":"https://orcid.org/0000-0002-7333-7830","observed_orcids":["https://orcid.org/0000-0002-7333-7830"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Banner, Ron","raw_affiliation_strings":["Habana Labs -An Intel company , Caesarea , Israel,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Habana Labs -An Intel company , Caesarea , Israel,","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025123281","display_name":"Daniel Soudry","orcid":"https://orcid.org/0000-0001-9368-6352","observed_orcids":["https://orcid.org/0000-0001-9368-6352"]},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Soudry, Daniel","raw_affiliation_strings":["Department of Electrical Engineering -Technion , Haifa , Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering -Technion , Haifa , Israel","institution_ids":["https://openalex.org/I174306211"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9789999723434448,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9789999723434448,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.007899999618530273,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/minimum-variance-unbiased-estimation","display_name":"minimum variance unbiased estimation","score":0.5460000038146973},{"id":"https://openalex.org/keywords/weight-pruning","display_name":"weight pruning","score":0.6370000243186951},{"id":"https://openalex.org/keywords/training-acceleration","display_name":"training acceleration","score":0.492000013589859},{"id":"https://openalex.org/keywords/mean-squared-error-minimization","display_name":"mean square error minimization","score":0.5249999761581421}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7066035866737366},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.660245954990387},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6319246292114258},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.6214834451675415},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.5910703539848328},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5320734977722168},{"id":"https://openalex.org/C139945424","wikidata":"https://www.wikidata.org/wiki/Q1940696","display_name":"Mean squared error","level":2,"score":0.5085800886154175},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.5085607767105103},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.43862104415893555},{"id":"https://openalex.org/C3832189","wikidata":"https://www.wikidata.org/wiki/Q8588916","display_name":"Models of neural computation","level":3,"score":0.4281972050666809},{"id":"https://openalex.org/C132943942","wikidata":"https://www.wikidata.org/wiki/Q2562511","display_name":"Footprint","level":2,"score":0.4248467683792114},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32585233449935913},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2568941116333008},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.09302753210067749},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2203.10991","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2203.10991","pdf_url":"https://arxiv.org/pdf/2203.10991","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2203.10991","doi":"https://doi.org/10.48550/arxiv.2203.10991","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2203.10991","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2203.10991","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2203.10991","pdf_url":"https://arxiv.org/pdf/2203.10991","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4221157729.pdf","grobid_xml":"https://content.openalex.org/works/W4221157729.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W2025713906","https://openalex.org/W2896409484","https://openalex.org/W2916954108","https://openalex.org/W2985339713","https://openalex.org/W3102093079","https://openalex.org/W3107472389","https://openalex.org/W3211043030"],"related_works":["https://openalex.org/W3092052590","https://openalex.org/W6885029860","https://openalex.org/W3132669550","https://openalex.org/W3034808851","https://openalex.org/W3127067080","https://openalex.org/W2995492258","https://openalex.org/W3207828344","https://openalex.org/W4320036900","https://openalex.org/W4414491954","https://openalex.org/W4366197771"],"abstract_inverted_index":{"In":[0],"deep":[1],"learning,":[2],"fine-grained":[3],"N:M":[4],"sparsity":[5,144,151],"reduces":[6],"the":[7,42,57,66,86,99,105,113,123,159],"data":[8],"footprint":[9],"and":[10,21,44,107,137,149],"bandwidth":[11],"of":[12,27,89,98,122],"a":[13,77],"General":[14],"Matrix":[15],"multiply":[16],"(GEMM)":[17],"up":[18,174],"to":[19,37,40,65,84,171],"x2,":[20],"doubles":[22],"throughput":[23],"by":[24],"skipping":[25],"computation":[26],"zero":[28],"values.":[29],"So":[30],"far,":[31],"it":[32,109],"was":[33],"mainly":[34],"only":[35],"used":[36,54],"prune":[38],"weights":[39,106],"accelerate":[41],"forward":[43],"backward":[45],"phases.":[46],"We":[47,93,132],"examine":[48],"how":[49],"this":[50,72,156],"method":[51],"can":[52],"be":[53],"also":[55],"for":[56,103,112,147],"neural":[58,68,114,124],"gradients":[59,62,125],"(i.e.,":[60],"loss":[61],"with":[63],"respect":[64],"intermediate":[67],"layer":[69],"outputs).":[70],"To":[71],"end,":[73],"we":[74,117,162],"first":[75],"establish":[76],"tensor-level":[78],"optimality":[79],"criteria.":[80],"Previous":[81],"works":[82,101],"aimed":[83],"minimize":[85],"mean-square-error":[87],"(MSE)":[88],"each":[90],"pruned":[91],"block.":[92],"show":[94,118],"that":[95,119,139],"while":[96],"minimization":[97],"MSE":[100],"fine":[102],"pruning":[104,121,130],"activations,":[108],"catastrophically":[110],"fails":[111],"gradients.":[115],"Instead,":[116],"accurate":[120],"requires":[126],"an":[127],"unbiased":[128],"minimum-variance":[129],"mask.":[131],"design":[133],"such":[134,166],"specialized":[135],"masks,":[136],"find":[138],"in":[140,169],"most":[141],"cases,":[142],"1:2":[143],"is":[145,152,157],"sufficient":[146],"training,":[148],"2:4":[150],"usually":[153],"enough":[154],"when":[155],"not":[158],"case.":[160],"Further,":[161],"suggest":[163],"combining":[164],"several":[165],"methods":[167],"together":[168],"order":[170],"potentially":[172],"speed":[173],"training":[175],"even":[176],"more.":[177]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":3}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2022-04-03T00:00:00"}