{"id":"https://openalex.org/W4388145401","doi":"https://doi.org/10.1109/tpami.2023.3329173","title":"MetaFormer Baselines for Vision","display_name":"MetaFormer Baselines for Vision","publication_year":2023,"publication_date":"2023-11-01","ids":{"openalex":"https://openalex.org/W4388145401","doi":"https://doi.org/10.1109/tpami.2023.3329173","pmid":"https://pubmed.ncbi.nlm.nih.gov/37910405"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2023.3329173","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2023.3329173","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044840682","display_name":"Weihao Yu","orcid":"https://orcid.org/0000-0003-3349-5890"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":true,"raw_author_name":"Weihao Yu","raw_affiliation_strings":["National University of Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023000066","display_name":"Chenyang Si","orcid":"https://orcid.org/0000-0002-3354-1968"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chenyang Si","raw_affiliation_strings":["Sea AI Lab, Singapore"],"affiliations":[{"raw_affiliation_string":"Sea AI Lab, Singapore","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010883708","display_name":"Pan Zhou","orcid":"https://orcid.org/0000-0003-3400-8943"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan Zhou","raw_affiliation_strings":["Sea AI Lab, Singapore"],"affiliations":[{"raw_affiliation_string":"Sea AI Lab, Singapore","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103400476","display_name":"Mi Luo","orcid":"https://orcid.org/0000-0002-8602-4681"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Mi Luo","raw_affiliation_strings":["National University of Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103181294","display_name":"Yichen Zhou","orcid":"https://orcid.org/0000-0002-0596-2087"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yichen Zhou","raw_affiliation_strings":["Sea AI Lab, Singapore"],"affiliations":[{"raw_affiliation_string":"Sea AI Lab, Singapore","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100668696","display_name":"Jiashi Feng","orcid":"https://orcid.org/0000-0001-6843-0064"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiashi Feng","raw_affiliation_strings":["Sea AI Lab, Singapore"],"affiliations":[{"raw_affiliation_string":"Sea AI Lab, Singapore","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100381753","display_name":"Shuicheng Yan","orcid":"https://orcid.org/0000-0001-8906-3777"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shuicheng Yan","raw_affiliation_strings":["Sea AI Lab, Singapore"],"affiliations":[{"raw_affiliation_string":"Sea AI Lab, Singapore","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015574447","display_name":"Xinchao Wang","orcid":"https://orcid.org/0000-0003-0057-1404"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xinchao Wang","raw_affiliation_strings":["National University of Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5044840682"],"corresponding_institution_ids":["https://openalex.org/I165932596"],"apc_list":null,"apc_paid":null,"fwci":27.1406,"has_fulltext":false,"cited_by_count":228,"citation_normalized_percentile":{"value":0.99784688,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"46","issue":"2","first_page":"896","last_page":"912"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.9635611772537231},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7534627318382263},{"id":"https://openalex.org/keywords/separable-space","display_name":"Separable space","score":0.5066235661506653},{"id":"https://openalex.org/keywords/suzuki-kasami-algorithm","display_name":"Suzuki-Kasami algorithm","score":0.4507583677768707},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4419686496257782},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3455073833465576},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.15127763152122498},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.12196710705757141}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.9635611772537231},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7534627318382263},{"id":"https://openalex.org/C70710897","wikidata":"https://www.wikidata.org/wiki/Q680081","display_name":"Separable space","level":2,"score":0.5066235661506653},{"id":"https://openalex.org/C18664526","wikidata":"https://www.wikidata.org/wiki/Q7651187","display_name":"Suzuki-Kasami algorithm","level":3,"score":0.4507583677768707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4419686496257782},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3455073833465576},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15127763152122498},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.12196710705757141},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2023.3329173","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2023.3329173","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:37910405","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37910405","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2531075042","display_name":null,"funder_award_id":"AISG2-RP-2021-023","funder_id":"https://openalex.org/F4320320709","funder_display_name":"National Research Foundation Singapore"},{"id":"https://openalex.org/G7078142857","display_name":null,"funder_award_id":"A-0005947-21-00","funder_id":"https://openalex.org/F4320320698","funder_display_name":"National University of Singapore"}],"funders":[{"id":"https://openalex.org/F4320320698","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49"},{"id":"https://openalex.org/F4320320709","display_name":"National Research Foundation Singapore","ror":"https://ror.org/03cpyc314"},{"id":"https://openalex.org/F4554660148","display_name":"Advanced Research and Technology Innovation Centre, College of Design and Engineering, National University of Singapore","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":132,"referenced_works":["https://openalex.org/W77200240","https://openalex.org/W1522301498","https://openalex.org/W1677182931","https://openalex.org/W1861492603","https://openalex.org/W2086161653","https://openalex.org/W2108598243","https://openalex.org/W2117539524","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2331143823","https://openalex.org/W2531409750","https://openalex.org/W2612083123","https://openalex.org/W2737258237","https://openalex.org/W2752782242","https://openalex.org/W2884822772","https://openalex.org/W2896457183","https://openalex.org/W2899663614","https://openalex.org/W2903739847","https://openalex.org/W2908510526","https://openalex.org/W2952809536","https://openalex.org/W2963150697","https://openalex.org/W2963163009","https://openalex.org/W2964241181","https://openalex.org/W2965373594","https://openalex.org/W2968124245","https://openalex.org/W2972451902","https://openalex.org/W2983180560","https://openalex.org/W2992308087","https://openalex.org/W2995435108","https://openalex.org/W2998508940","https://openalex.org/W3012740297","https://openalex.org/W3015468748","https://openalex.org/W3016129867","https://openalex.org/W3034429256","https://openalex.org/W3035452548","https://openalex.org/W3035682985","https://openalex.org/W3037492894","https://openalex.org/W3090449556","https://openalex.org/W3092879656","https://openalex.org/W3094502228","https://openalex.org/W3096609285","https://openalex.org/W3097777922","https://openalex.org/W3103334733","https://openalex.org/W3121523901","https://openalex.org/W3131500599","https://openalex.org/W3138516171","https://openalex.org/W3162090017","https://openalex.org/W3163465952","https://openalex.org/W3170841864","https://openalex.org/W3177096435","https://openalex.org/W3177183540","https://openalex.org/W3186979696","https://openalex.org/W3196691776","https://openalex.org/W3205328383","https://openalex.org/W4214634256","https://openalex.org/W4221166842","https://openalex.org/W4224308101","https://openalex.org/W4226272139","https://openalex.org/W4226297238","https://openalex.org/W4226363321","https://openalex.org/W4229010634","https://openalex.org/W4285704109","https://openalex.org/W4286914341","https://openalex.org/W4288089799","https://openalex.org/W4288404646","https://openalex.org/W4288581820","https://openalex.org/W4288804646","https://openalex.org/W4289752563","https://openalex.org/W4308558335","https://openalex.org/W4312443924","https://openalex.org/W4312769131","https://openalex.org/W4312820606","https://openalex.org/W4312847199","https://openalex.org/W4312957757","https://openalex.org/W4313007769","https://openalex.org/W4313160444","https://openalex.org/W4320167334","https://openalex.org/W4366352791","https://openalex.org/W4385245566","https://openalex.org/W4385346076","https://openalex.org/W4385656546","https://openalex.org/W4386075553","https://openalex.org/W6631190155","https://openalex.org/W6684191040","https://openalex.org/W6739879593","https://openalex.org/W6755207826","https://openalex.org/W6755977528","https://openalex.org/W6757555829","https://openalex.org/W6757585730","https://openalex.org/W6757817989","https://openalex.org/W6758139636","https://openalex.org/W6762287338","https://openalex.org/W6763468762","https://openalex.org/W6766673545","https://openalex.org/W6767279747","https://openalex.org/W6769607603","https://openalex.org/W6769627184","https://openalex.org/W6776048684","https://openalex.org/W6776684981","https://openalex.org/W6778883912","https://openalex.org/W6779879114","https://openalex.org/W6780226713","https://openalex.org/W6784333009","https://openalex.org/W6788135285","https://openalex.org/W6790690058","https://openalex.org/W6791364533","https://openalex.org/W6793164127","https://openalex.org/W6795140394","https://openalex.org/W6796417832","https://openalex.org/W6796931752","https://openalex.org/W6797578546","https://openalex.org/W6798107787","https://openalex.org/W6799423381","https://openalex.org/W6800735434","https://openalex.org/W6801441846","https://openalex.org/W6801655670","https://openalex.org/W6802297474","https://openalex.org/W6803680838","https://openalex.org/W6809468216","https://openalex.org/W6810081322","https://openalex.org/W6810661123","https://openalex.org/W6810859482","https://openalex.org/W6810964969","https://openalex.org/W6811002592","https://openalex.org/W6811173682","https://openalex.org/W6838697126","https://openalex.org/W6839692567","https://openalex.org/W6839715048","https://openalex.org/W6839818334","https://openalex.org/W6840140340","https://openalex.org/W6840866624","https://openalex.org/W6848935878"],"related_works":["https://openalex.org/W2136386550","https://openalex.org/W2080520651","https://openalex.org/W2123031170","https://openalex.org/W1598084473","https://openalex.org/W2466134378","https://openalex.org/W1824315176","https://openalex.org/W1961065714","https://openalex.org/W1577962751","https://openalex.org/W2783967463","https://openalex.org/W3161507817"],"abstract_inverted_index":{"MetaFormer,":[0,27,260],"the":[1,24,35,47,78,81,104,115,155,164,171,177,180,192,215,222,225],"abstracted":[2],"architecture":[3],"of":[4,26,70,122,130,163,240,272,287],"Transformer,":[5],"has":[6],"been":[7],"found":[8],"to":[9,112,258,291,298],"play":[10],"a":[11,109,230,265,285],"significant":[12],"role":[13],"in":[14,214,221,302],"achieving":[15],"competitive":[16],"performance.":[17,57,71,281],"In":[18,255],"this":[19],"paper,":[20],"we":[21,39,261],"further":[22],"explore":[23],"capacity":[25],"again,":[28],"by":[29],"migrating":[30],"our":[31,60,256],"focus":[32],"away":[33],"from":[34,158],"token":[36,79,100,105,135,148,178,212],"mixer":[37,106,213],"design:":[38],"introduce":[40],"several":[41],"baseline":[42],"models":[43,156,305,312],"under":[44,246],"MetaFormer":[45,65,82,95,140,159],"using":[46],"most":[48],"basic":[49],"or":[50,253],"common":[51,172],"mixers,":[52],"and":[53,218,311],"demonstrate":[54],"their":[55],"gratifying":[56],"We":[58,295],"summarize":[59],"observations":[61],"as":[62,77,107,176,188,211],"follows:":[63],"(1)":[64],"ensures":[66],"solid":[67],"lower":[68],"bound":[69],"By":[72,205],"merely":[73],"adopting":[74],"identity":[75],"mapping":[76],"mixer,":[80,179],"model,":[83],"termed":[84,182],"IdentityFormer,":[85],"achieves":[86,237,279],"[Formula:":[87,123],"see":[88,124],"text]80%":[89],"accuracy":[90,121,239],"on":[91,202,233],"ImageNet-1":[92,203,234],"K.":[93,204],"(2)":[94],"works":[96],"well":[97],"with":[98,275],"arbitrary":[99],"mixers.":[101],"When":[102],"specifying":[103],"even":[108],"random":[110],"matrix":[111],"mix":[113],"tokens,":[114],"resulting":[116,226],"model":[117,181,195,227],"RandFormer":[118],"yields":[119],"an":[120,238],"text]81%,":[125],"outperforming":[126],"IdentityFormer.":[127],"Rest":[128],"assured":[129],"MetaFormer's":[131],"results":[132],"when":[133],"new":[134,200,231,266],"mixers":[136,149],"are":[137,313],"adopted.":[138],"(3)":[139],"effortlessly":[141],"offers":[142],"state-of-the-art":[143],"results.":[144],"With":[145],"just":[146],"conventional":[147],"dated":[150],"back":[151],"five":[152],"years":[153],"ago,":[154],"instantiated":[157],"already":[160],"beat":[161],"state":[162],"art.":[165],"(a)":[166],"ConvFormer":[167],"outperforms":[168,191],"ConvNeXt.":[169,196],"Taking":[170],"depthwise":[173,208],"separable":[174,209],"convolutions":[175,210],"ConvFormer,":[183],"which":[184],"can":[185],"be":[186],"regarded":[187],"pure":[189],"CNNs,":[190],"strong":[193],"CNN":[194],"(b)":[197],"CAFormer":[198,228],"sets":[199,229],"record":[201,232],"simply":[206],"applying":[207],"bottom":[216],"stages":[217],"vanilla":[219],"self-attention":[220],"top":[223],"stages,":[224],"K:":[235],"it":[236],"85.5%":[241],"at":[242,315],"224":[243],"\u00d7224":[244],"resolution,":[245],"normal":[247],"supervised":[248],"training":[249],"without":[250],"external":[251],"data":[252],"distillation.":[254],"expedition":[257],"probe":[259],"also":[262],"find":[263,299],"that":[264],"activation,":[267],"StarReLU,":[268],"reduces":[269],"71%":[270],"FLOPs":[271],"activation":[273],"compared":[274],"commonly-used":[276],"GELU":[277],"yet":[278],"better":[280],"Specifically,":[282],"StarReLU":[283,297],"is":[284],"variant":[286],"Squared":[288],"ReLU":[289],"dedicated":[290],"alleviating":[292],"distribution":[293],"shift.":[294],"expect":[296],"great":[300],"potential":[301],"MetaFormer-":[303],"like":[304],"alongside":[306],"other":[307],"neural":[308],"networks.":[309],"Code":[310],"available":[314],"https://github.com/sail-sg/metaformer.":[316]},"counts_by_year":[{"year":2026,"cited_by_count":41},{"year":2025,"cited_by_count":130},{"year":2024,"cited_by_count":51},{"year":2023,"cited_by_count":6}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}