{"id":"https://openalex.org/W2995333370","doi":"https://doi.org/10.48550/arxiv.2001.06838","title":"Towards Stabilizing Batch Statistics in Backward Propagation of Batch Normalization","display_name":"Towards Stabilizing Batch Statistics in Backward Propagation of Batch Normalization","publication_year":2020,"publication_date":"2020-01-19","ids":{"openalex":"https://openalex.org/W2995333370","doi":"https://doi.org/10.48550/arxiv.2001.06838","mag":"2995333370"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2001.06838","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.06838","pdf_url":"https://arxiv.org/pdf/2001.06838","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2001.06838","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Yan, Junjie","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan, Junjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010826431","display_name":"Ruosi Wan","orcid":"https://orcid.org/0000-0003-2115-6621","observed_orcids":["https://orcid.org/0000-0003-2115-6621"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wan, Ruosi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100362465","display_name":"Xiangyu Zhang","orcid":"https://orcid.org/0000-0003-2138-4608","observed_orcids":["https://orcid.org/0000-0003-2138-4608"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Xiangyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101423831","display_name":"Wei Zhang","orcid":"https://orcid.org/0000-0001-6496-849X","observed_orcids":["https://orcid.org/0000-0001-6496-849X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101959185","display_name":"Yichen Wei","orcid":"https://orcid.org/0000-0002-3593-8280","observed_orcids":["https://orcid.org/0000-0002-3593-8280"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Yichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100785015","display_name":"Jian Sun","orcid":"https://orcid.org/0000-0002-6178-4166","observed_orcids":["https://orcid.org/0000-0002-6178-4166","https://orcid.org/0009-0003-4125-9764"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":28,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7470999956130981,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7470999956130981,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.14669999480247498,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.03929999843239784,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/batch-normalization","display_name":"batch normalization","score":0.9900000095367432},{"id":"https://openalex.org/keywords/backward-propagation","display_name":"backward propagation","score":0.7789999842643738},{"id":"https://openalex.org/keywords/small-batch-sizes","display_name":"small batch sizes","score":0.9200000166893005},{"id":"https://openalex.org/keywords/normalization-methods","display_name":"normalization methods","score":0.47600001096725464},{"id":"https://openalex.org/keywords/computer-vision","display_name":"computer vision","score":0.5799999833106995},{"id":"https://openalex.org/keywords/object-detection","display_name":"object detection","score":0.6290000081062317},{"id":"https://openalex.org/keywords/image-segmentation","display_name":"image segmentation","score":0.5879999995231628},{"id":"https://openalex.org/keywords/inference-efficiency","display_name":"inference efficiency","score":0.531000018119812}],"concepts":[{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.7557306289672852},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5668155550956726},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.43659836053848267},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.40461266040802},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.36653417348861694},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.057329654693603516},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2001.06838","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.06838","pdf_url":"https://arxiv.org/pdf/2001.06838","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3002418729","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2001.06838","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2001.06838","doi":"https://doi.org/10.48550/arxiv.2001.06838","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2001.06838","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2001.06838","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.06838","pdf_url":"https://arxiv.org/pdf/2001.06838","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1677182931","https://openalex.org/W1836465849","https://openalex.org/W1861492603","https://openalex.org/W2117539524","https://openalex.org/W2194775991","https://openalex.org/W2340897893","https://openalex.org/W2412782625","https://openalex.org/W2502312327","https://openalex.org/W2560023338","https://openalex.org/W2565639579","https://openalex.org/W2588610957","https://openalex.org/W2921569601","https://openalex.org/W2937220865","https://openalex.org/W2963016543","https://openalex.org/W2963150697","https://openalex.org/W2963351448","https://openalex.org/W2991391304"],"related_works":["https://openalex.org/W3011538607","https://openalex.org/W2787911558","https://openalex.org/W2937220865","https://openalex.org/W3112509143","https://openalex.org/W2786689170","https://openalex.org/W3092408118","https://openalex.org/W3210541621","https://openalex.org/W3174135168","https://openalex.org/W2795783309","https://openalex.org/W2948330917"],"abstract_inverted_index":{"Batch":[0,142],"Normalization":[1,143],"(BN)":[2],"is":[3,44],"one":[4],"of":[5,30,51,69,102,125,151,171,184],"the":[6,28,49,67,123,149,169,182],"most":[7],"widely":[8],"used":[9],"techniques":[10,58],"in":[11,79,99,154,164,186,200],"Deep":[12],"Learning":[13],"field.":[14],"But":[15],"its":[16],"performance":[17,68,150],"can":[18,120,146],"awfully":[19],"degrade":[20],"with":[21,117],"insufficient":[22],"batch":[23,42,96,114,156],"size.":[24],"This":[25],"weakness":[26],"limits":[27],"usage":[29],"BN":[31,70,153],"on":[32,104,130],"many":[33,55],"computer":[34,188],"vision":[35,189],"tasks":[36,190],"like":[37],"detection":[38],"or":[39,72],"segmentation,":[40],"where":[41],"size":[43],"usually":[45],"small":[46,155],"due":[47],"to":[48,65,74],"constraint":[50],"memory":[52],"consumption.":[53,85],"Therefore":[54],"modified":[56],"normalization":[57,137],"have":[59,73],"been":[60,108,198],"proposed,":[61],"which":[62,105],"either":[63],"fail":[64],"restore":[66,148],"completely,":[71],"introduce":[75],"additional":[76,161],"nonlinear":[77,162],"operations":[78,163],"inference":[80,165],"procedure":[81],"and":[82,177,193],"increase":[83],"huge":[84],"In":[86],"this":[87],"paper,":[88],"we":[89,133],"reveal":[90],"that":[91],"there":[92],"are":[93],"two":[94],"extra":[95,113],"statistics":[97,115],"involved":[98],"backward":[100],"propagation":[101],"BN,":[103],"has":[106,197],"never":[107],"well":[109],"discussed":[110],"before.":[111],"The":[112,195],"associated":[116],"gradients":[118],"also":[119],"severely":[121],"affect":[122],"training":[124],"deep":[126],"neural":[127],"network.":[128],"Based":[129],"our":[131],"analysis,":[132],"propose":[134],"a":[135],"novel":[136],"method,":[138],"named":[139],"Moving":[140],"Average":[141],"(MABN).":[144],"MABN":[145,172,185],"completely":[147],"vanilla":[152],"cases,":[157],"without":[158],"introducing":[159],"any":[160],"procedure.":[166],"We":[167],"prove":[168],"benefits":[170],"by":[173],"both":[174],"theoretical":[175],"analysis":[176],"experiments.":[178],"Our":[179],"experiments":[180],"demonstrate":[181],"effectiveness":[183],"multiple":[187],"including":[191],"ImageNet":[192],"COCO.":[194],"code":[196],"released":[199],"https://github.com/megvii-model/MABN.":[201]},"counts_by_year":[{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":5}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2019-12-26T00:00:00"}