{"id":"https://openalex.org/W7162635707","doi":"https://doi.org/10.48550/arxiv.2605.27541","title":"SparseOpt: Addressing Normalization-induced Gradient Skew in Sparse Training","display_name":"SparseOpt: Addressing Normalization-induced Gradient Skew in Sparse Training","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162635707","doi":"https://doi.org/10.48550/arxiv.2605.27541"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.27541","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27541","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.27541","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137301631","display_name":"Mohammed Adnan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Adnan, Mohammed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136724494","display_name":"Rohan Jain","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jain, Rohan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109597158","display_name":"Tom Jacobs","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jacobs, Tom","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137280382","display_name":"Ekansh Sharma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sharma, Ekansh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073514348","display_name":"Rahul G. Krishnan","orcid":"https://orcid.org/0000-0002-7955-3956"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krishnan, Rahul G.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137224257","display_name":"Rebekka Burkholz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Burkholz, Rebekka","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137215114","display_name":"Yani Ioannou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ioannou, Yani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.4388999938964844,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.4388999938964844,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.3702000081539154,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.04820000007748604,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.703000009059906},{"id":"https://openalex.org/keywords/skew","display_name":"Skew","score":0.5501000285148621},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5490000247955322},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4941999912261963},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.49070000648498535},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4625000059604645},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4528000056743622},{"id":"https://openalex.org/keywords/sparse-approximation","display_name":"Sparse approximation","score":0.39399999380111694}],"concepts":[{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.703000009059906},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6929000020027161},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5823000073432922},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.5501000285148621},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5490000247955322},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4941999912261963},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.49070000648498535},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46939998865127563},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4625000059604645},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4528000056743622},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3986000120639801},{"id":"https://openalex.org/C124066611","wikidata":"https://www.wikidata.org/wiki/Q28684319","display_name":"Sparse approximation","level":2,"score":0.39399999380111694},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.3871000111103058},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.37630000710487366},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.3695000112056732},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.3546999990940094},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.3280999958515167},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.2678000032901764},{"id":"https://openalex.org/C88548561","wikidata":"https://www.wikidata.org/wiki/Q347599","display_name":"sort","level":2,"score":0.26159998774528503}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.27541","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27541","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.27541","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27541","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Dynamic":[0],"Sparse":[1],"Training":[2],"(DST)":[3],"methods":[4,24],"train":[5],"neural":[6],"networks":[7],"by":[8],"maintaining":[9],"sparsity":[10],"while":[11],"dynamically":[12],"adapting":[13],"the":[14,18,85,96,101],"network":[15],"topology.":[16],"Despite":[17],"promise":[19],"of":[20,87,100],"reduced":[21],"computation,":[22],"DST":[23,116],"converge":[25],"significantly":[26],"slower":[27],"than":[28],"dense":[29,120],"training,":[30,53],"often":[31],"requiring":[32],"comparable":[33],"training":[34,93],"time":[35],"to":[36,60],"achieve":[37],"similar":[38],"accuracy.":[39],"We":[40],"demonstrate":[41,71],"both":[42],"analytically":[43],"and":[44,54,69,75,94,108],"empirically":[45],"that":[46],"Batch":[47,104],"Normalization":[48],"(BN)":[49],"adversely":[50],"affects":[51],"sparse":[52,92,106],"propose":[55],"SparseOpt,":[56],"a":[57,111],"sparsity-aware":[58],"optimizer,":[59],"address":[61],"this.":[62],"Experiments":[63],"on":[64],"ResNet":[65],"models":[66],"across":[67],"CIFAR-100":[68],"ImageNet":[70],"consistently":[72],"faster":[73],"convergence":[74],"improved":[76],"generalization":[77],"with":[78,119],"our":[79],"proposed":[80],"method.":[81],"Our":[82],"work":[83],"highlights":[84],"limitations":[86],"current":[88],"normalization":[89],"layers":[90],"in":[91],"provides":[95],"first":[97],"systematic":[98],"study":[99],"interaction":[102],"between":[103],"Normalization,":[105],"layers,":[107],"DST,":[109],"taking":[110],"significant":[112],"step":[113],"toward":[114],"making":[115],"practically":[117],"competitive":[118],"training.":[121]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-29T00:00:00"}