{"id":"https://openalex.org/W4289785667","doi":"https://doi.org/10.1145/3503161.3547860","title":"Unified Normalization for Accelerating and Stabilizing Transformers","display_name":"Unified Normalization for Accelerating and Stabilizing Transformers","publication_year":2022,"publication_date":"2022-10-10","ids":{"openalex":"https://openalex.org/W4289785667","doi":"https://doi.org/10.1145/3503161.3547860"},"language":"en","primary_location":{"id":"doi:10.1145/3503161.3547860","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3503161.3547860","pdf_url":null,"source":{"id":"https://openalex.org/S4363608757","display_name":"Proceedings of the 30th ACM International Conference on Multimedia","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2208.01313","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100650556","display_name":"Qiming Yang","orcid":"https://orcid.org/0009-0002-5709-6798"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Qiming Yang","raw_affiliation_strings":["Hikvision Research Institute, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hikvision Research Institute, Hangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100324058","display_name":"Kai Zhang","orcid":"https://orcid.org/0000-0003-4058-6512"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kai Zhang","raw_affiliation_strings":["Hikvision Research Institute, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hikvision Research Institute, Hangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051729395","display_name":"Chaoxiang Lan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chaoxiang Lan","raw_affiliation_strings":["Hikvision Research Institute, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hikvision Research Institute, Hangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100355790","display_name":"Zhi Yang","orcid":"https://orcid.org/0000-0002-0871-5882"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhi Yang","raw_affiliation_strings":["Hikvision Research Institute, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hikvision Research Institute, Hangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046994319","display_name":"Zheyang Li","orcid":"https://orcid.org/0000-0002-0229-8707"},"institutions":[{"id":"https://openalex.org/I4210132223","display_name":"Zhejiang Radio and Television University","ror":"https://ror.org/02kpaqs60","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210132223"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheyang Li","raw_affiliation_strings":["Hikvision Research Institute & Zhejiang University, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hikvision Research Institute & Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I4210132223","https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031951156","display_name":"Wenming Tan","orcid":"https://orcid.org/0000-0003-1338-4536"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wenming Tan","raw_affiliation_strings":["Hikvision Research Institute, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hikvision Research Institute, Hangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101485989","display_name":"Jun Xiao","orcid":"https://orcid.org/0000-0002-6142-9914"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Xiao","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085955762","display_name":"Shiliang Pu","orcid":"https://orcid.org/0000-0001-5269-7821"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shiliang Pu","raw_affiliation_strings":["Hikvision Research Institute, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hikvision Research Institute, Hangzhou, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5100650556"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0598,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.25635202,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"4445","last_page":"4455"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7314013242721558},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.619307816028595},{"id":"https://openalex.org/keywords/inefficiency","display_name":"Inefficiency","score":0.5991336107254028},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5812480449676514},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5497975945472717},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.45935845375061035},{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.44555145502090454},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.41886845231056213},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.40631526708602905},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3515225648880005},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.21002471446990967},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.1456083059310913}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7314013242721558},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.619307816028595},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.5991336107254028},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5812480449676514},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5497975945472717},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.45935845375061035},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.44555145502090454},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.41886845231056213},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.40631526708602905},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3515225648880005},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.21002471446990967},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.1456083059310913},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3503161.3547860","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3503161.3547860","pdf_url":null,"source":{"id":"https://openalex.org/S4363608757","display_name":"Proceedings of the 30th ACM International Conference on Multimedia","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2208.01313","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.01313","pdf_url":"https://arxiv.org/pdf/2208.01313","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2208.01313","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.01313","pdf_url":"https://arxiv.org/pdf/2208.01313","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1593271688","https://openalex.org/W1861492603","https://openalex.org/W2060370382","https://openalex.org/W2108598243","https://openalex.org/W2565639579","https://openalex.org/W2921569601","https://openalex.org/W2933138175","https://openalex.org/W2962780374","https://openalex.org/W2963150697","https://openalex.org/W3096609285","https://openalex.org/W3103334733","https://openalex.org/W3121523901","https://openalex.org/W3138516171","https://openalex.org/W3145450063","https://openalex.org/W3197478142","https://openalex.org/W3214897340","https://openalex.org/W4214588794","https://openalex.org/W4214669216","https://openalex.org/W4232212680","https://openalex.org/W4250482878","https://openalex.org/W4312349930","https://openalex.org/W4313007769"],"related_works":["https://openalex.org/W2058965144","https://openalex.org/W2164382479","https://openalex.org/W2146343568","https://openalex.org/W98480971","https://openalex.org/W2150291671","https://openalex.org/W2013643406","https://openalex.org/W2027972911","https://openalex.org/W2157978810","https://openalex.org/W2597809628","https://openalex.org/W3046370962"],"abstract_inverted_index":{"Solid":[0],"results":[1,68],"from":[2],"Transformers":[3,202],"have":[4],"made":[5],"them":[6],"prevailing":[7],"architectures":[8],"in":[9,20,40,69,74,158,168],"various":[10],"natural":[11],"language":[12,188],"and":[13,46,94,121,138,165,189,211],"vision":[14,190],"tasks.":[15,191],"As":[16],"a":[17,142],"default":[18],"component":[19],"Transformers,":[21],"Layer":[22],"Normalization":[23,106],"(LN)":[24],"normalizes":[25],"activations":[26],"within":[27],"each":[28],"token":[29],"to":[30,51,131,155,181],"boost":[31,132],"the":[32,112,136,195],"robustness.":[33],"However,":[34],"LN":[35,59,182],"requires":[36],"on-the-fly":[37],"statistics":[38,140],"calculation":[39],"inference":[41,113,209],"as":[42,44],"well":[43],"division":[45],"square":[47],"root":[48],"operations,":[49],"leading":[50],"inefficiency":[52],"on":[53,125,187,200],"hardware.":[54],"What":[55],"is":[56,81,153,162],"more,":[57],"replacing":[58],"with":[60,117,127,141,204],"other":[61,118],"hardware-efficient":[62],"normalization":[63],"schemes":[64],"(e.g.,":[65],"Batch":[66],"Normalization)":[67],"inferior":[70],"performance,":[71],"even":[72],"collapse":[73,157],"training.":[75],"We":[76,171],"find":[77],"that":[78,173],"this":[79,169],"dilemma":[80],"caused":[82],"by":[83,114,134,183],"abnormal":[84],"behaviors":[85],"of":[86,197],"activation":[87,137],"statistics,":[88],"including":[89],"large":[90],"fluctuations":[91],"over":[92],"iterations":[93],"extreme":[95],"outliers":[96],"across":[97],"layers.":[98],"To":[99],"tackle":[100],"these":[101],"issues,":[102],"we":[103,193],"propose":[104],"Unified":[105],"(UN),":[107],"which":[108],"can":[109,175],"speed":[110],"up":[111],"being":[115],"fused":[116],"linear":[119],"operations":[120],"achieve":[122],"comparable":[123],"performance":[124,133],"par":[126],"LN.":[128],"UN":[129,174,205],"strives":[130],"calibrating":[135],"gradient":[139],"tailored":[143],"fluctuation":[144],"smoothing":[145],"strategy.":[146],"Meanwhile,":[147],"an":[148,177],"adaptive":[149],"outlier":[150],"filtration":[151],"strategy":[152],"applied":[154],"avoid":[156],"training":[159],"whose":[160],"effectiveness":[161],"theoretically":[163],"proved":[164],"experimentally":[166],"verified":[167],"paper.":[170],"demonstrate":[172],"be":[176,218],"efficient":[178],"drop-in":[179],"alternative":[180],"conducting":[184],"extensive":[185],"experiments":[186],"Besides,":[192],"evaluate":[194],"efficiency":[196],"our":[198],"method":[199],"GPU.":[201],"equipped":[203],"enjoy":[206],"about":[207],"31%":[208],"speedup":[210],"nearly":[212],"18%":[213],"memory":[214],"reduction.":[215],"Code":[216],"will":[217],"released":[219],"at":[220],"https://github.com/hikvision-research/Unified-Normalization.":[221]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}