{"id":"https://openalex.org/W4382599796","doi":"https://doi.org/10.3389/fcomp.2023.1178450","title":"Self-attention in vision transformers performs perceptual grouping, not attention","display_name":"Self-attention in vision transformers performs perceptual grouping, not attention","publication_year":2023,"publication_date":"2023-06-29","ids":{"openalex":"https://openalex.org/W4382599796","doi":"https://doi.org/10.3389/fcomp.2023.1178450"},"language":"en","primary_location":{"id":"doi:10.3389/fcomp.2023.1178450","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fcomp.2023.1178450","pdf_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1178450/pdf","source":{"id":"https://openalex.org/S4210211086","display_name":"Frontiers in Computer Science","issn_l":"2624-9898","issn":["2624-9898"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Computer Science","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1178450/pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066048483","display_name":"Paria Mehrani","orcid":"https://orcid.org/0000-0002-5566-9129"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Paria Mehrani","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science, York University, Toronto, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science, York University, Toronto, ON, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072681668","display_name":"John K. Tsotsos","orcid":"https://orcid.org/0000-0002-8621-9147"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"John K. Tsotsos","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science, York University, Toronto, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science, York University, Toronto, ON, Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5066048483"],"corresponding_institution_ids":["https://openalex.org/I192455969"],"apc_list":{"value":1150,"currency":"USD","value_usd":1285},"apc_paid":{"value":1150,"currency":"USD","value_usd":1285},"fwci":3.7124,"has_fulltext":true,"cited_by_count":38,"citation_normalized_percentile":{"value":0.94804314,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"5","issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11094","display_name":"Face Recognition and Perception","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10427","display_name":"Visual perception and processing mechanisms","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5539817810058594},{"id":"https://openalex.org/keywords/salience","display_name":"Salience (neuroscience)","score":0.5363631844520569},{"id":"https://openalex.org/keywords/visual-perception","display_name":"Visual perception","score":0.5324208736419678},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5239819884300232},{"id":"https://openalex.org/keywords/visual-attention","display_name":"Visual attention","score":0.5041278600692749},{"id":"https://openalex.org/keywords/computational-model","display_name":"Computational model","score":0.49410465359687805},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.49398189783096313},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.48090624809265137},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.4233779311180115},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.3642493486404419},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2912457585334778},{"id":"https://openalex.org/keywords/neuroscience","display_name":"Neuroscience","score":0.11520257592201233},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0975823700428009}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5539817810058594},{"id":"https://openalex.org/C108154423","wikidata":"https://www.wikidata.org/wiki/Q1469792","display_name":"Salience (neuroscience)","level":2,"score":0.5363631844520569},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.5324208736419678},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5239819884300232},{"id":"https://openalex.org/C2986089797","wikidata":"https://www.wikidata.org/wiki/Q6501338","display_name":"Visual attention","level":3,"score":0.5041278600692749},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.49410465359687805},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.49398189783096313},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.48090624809265137},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.4233779311180115},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3642493486404419},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2912457585334778},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.11520257592201233},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0975823700428009},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3389/fcomp.2023.1178450","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fcomp.2023.1178450","pdf_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1178450/pdf","source":{"id":"https://openalex.org/S4210211086","display_name":"Frontiers in Computer Science","issn_l":"2624-9898","issn":["2624-9898"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Computer Science","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:955c4b45b16d4e438219afec3d6bd490","is_oa":true,"landing_page_url":"https://doaj.org/article/955c4b45b16d4e438219afec3d6bd490","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Frontiers in Computer Science, Vol 5 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3389/fcomp.2023.1178450","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fcomp.2023.1178450","pdf_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1178450/pdf","source":{"id":"https://openalex.org/S4210211086","display_name":"Frontiers in Computer Science","issn_l":"2624-9898","issn":["2624-9898"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Computer Science","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8864442964","display_name":null,"funder_award_id":"950-231659","funder_id":"https://openalex.org/F4320320994","funder_display_name":"Canada Research Chairs"}],"funders":[{"id":"https://openalex.org/F4320320994","display_name":"Canada Research Chairs","ror":"https://ror.org/0517h6h17"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4382599796.pdf"},"referenced_works_count":136,"referenced_works":["https://openalex.org/W119228284","https://openalex.org/W177844970","https://openalex.org/W248046807","https://openalex.org/W1514535095","https://openalex.org/W1971534008","https://openalex.org/W1985393502","https://openalex.org/W1998938084","https://openalex.org/W2006063316","https://openalex.org/W2013894622","https://openalex.org/W2014886758","https://openalex.org/W2020825481","https://openalex.org/W2025819347","https://openalex.org/W2038220177","https://openalex.org/W2040036684","https://openalex.org/W2040568291","https://openalex.org/W2095627201","https://openalex.org/W2103147032","https://openalex.org/W2112845172","https://openalex.org/W2118615399","https://openalex.org/W2138046011","https://openalex.org/W2139748113","https://openalex.org/W2143237238","https://openalex.org/W2152669853","https://openalex.org/W2156047020","https://openalex.org/W2164084182","https://openalex.org/W2164364459","https://openalex.org/W2165728411","https://openalex.org/W2166206801","https://openalex.org/W2171590216","https://openalex.org/W2175160520","https://openalex.org/W2194775991","https://openalex.org/W2295763703","https://openalex.org/W2343204383","https://openalex.org/W2537084945","https://openalex.org/W2552164092","https://openalex.org/W2612573399","https://openalex.org/W2744872705","https://openalex.org/W2792911100","https://openalex.org/W2808580750","https://openalex.org/W2939218116","https://openalex.org/W2949847915","https://openalex.org/W2950225513","https://openalex.org/W2952671852","https://openalex.org/W2971464184","https://openalex.org/W2979826702","https://openalex.org/W3025069833","https://openalex.org/W3034236144","https://openalex.org/W3035422918","https://openalex.org/W3089527440","https://openalex.org/W3094502228","https://openalex.org/W3119948327","https://openalex.org/W3121523901","https://openalex.org/W3137278571","https://openalex.org/W3138516171","https://openalex.org/W3143373604","https://openalex.org/W3145158551","https://openalex.org/W3145185940","https://openalex.org/W3146620971","https://openalex.org/W3151130473","https://openalex.org/W3159481202","https://openalex.org/W3161120562","https://openalex.org/W3164024107","https://openalex.org/W3170642968","https://openalex.org/W3170863103","https://openalex.org/W3171087525","https://openalex.org/W3172509117","https://openalex.org/W3176860045","https://openalex.org/W3192770954","https://openalex.org/W3195282418","https://openalex.org/W3200213953","https://openalex.org/W3204538018","https://openalex.org/W3205725084","https://openalex.org/W3209987414","https://openalex.org/W3213783001","https://openalex.org/W4210433173","https://openalex.org/W4212774754","https://openalex.org/W4212942565","https://openalex.org/W4213019189","https://openalex.org/W4213083558","https://openalex.org/W4214493665","https://openalex.org/W4214713996","https://openalex.org/W4225088380","https://openalex.org/W4234561751","https://openalex.org/W4251073084","https://openalex.org/W4281480780","https://openalex.org/W4282933246","https://openalex.org/W4286218466","https://openalex.org/W4287022992","https://openalex.org/W4289173098","https://openalex.org/W4291792830","https://openalex.org/W4295951228","https://openalex.org/W4297812995","https://openalex.org/W4298171088","https://openalex.org/W4299802238","https://openalex.org/W4306802685","https://openalex.org/W4312950730","https://openalex.org/W4321504988","https://openalex.org/W4360995299","https://openalex.org/W4384663241","https://openalex.org/W4385245566","https://openalex.org/W6630875275","https://openalex.org/W6680565713","https://openalex.org/W6685621245","https://openalex.org/W6687483927","https://openalex.org/W6732958910","https://openalex.org/W6737912528","https://openalex.org/W6739901393","https://openalex.org/W6761473936","https://openalex.org/W6769243733","https://openalex.org/W6769955919","https://openalex.org/W6776793684","https://openalex.org/W6777058603","https://openalex.org/W6784333009","https://openalex.org/W6788477181","https://openalex.org/W6789425149","https://openalex.org/W6792155083","https://openalex.org/W6792566387","https://openalex.org/W6792881003","https://openalex.org/W6794166682","https://openalex.org/W6794655914","https://openalex.org/W6795297120","https://openalex.org/W6795475546","https://openalex.org/W6796761347","https://openalex.org/W6796931752","https://openalex.org/W6799166919","https://openalex.org/W6799770928","https://openalex.org/W6800018832","https://openalex.org/W6800217721","https://openalex.org/W6802741100","https://openalex.org/W6802910155","https://openalex.org/W6803680838","https://openalex.org/W6810938606","https://openalex.org/W6811002592","https://openalex.org/W6839026099","https://openalex.org/W6847279771","https://openalex.org/W6963229073"],"related_works":["https://openalex.org/W3089898354","https://openalex.org/W580018783","https://openalex.org/W3108188144","https://openalex.org/W2047972606","https://openalex.org/W2003057156","https://openalex.org/W4225895457","https://openalex.org/W2080638739","https://openalex.org/W4200100213","https://openalex.org/W2007643044","https://openalex.org/W2156012987"],"abstract_inverted_index":{"Recently,":[0],"a":[1,96,154,181],"considerable":[2],"number":[3],"of":[4,35,43,99,126,156,173,188,202,234],"studies":[5],"in":[6,18,46,61,70,83,132,143,153,167,180,211,247],"computer":[7],"vision":[8,14,47,62,127,157,248],"involve":[9],"deep":[10],"neural":[11],"architectures":[12],"called":[13],"transformers.":[15,158],"Visual":[16],"processing":[17],"these":[19,84,93,133,147,195],"models":[20,23,85,94,134,196],"incorporates":[21],"computational":[22,238],"that":[24,37,88,112,130,162,217,244],"are":[25],"claimed":[26],"to":[27,39,208,227],"implement":[28],"attention":[29,44,59,81,115,131,221],"mechanisms.":[30],"Despite":[31],"an":[32,186],"increasing":[33],"body":[34],"work":[36],"attempts":[38],"understand":[40],"the":[41,58,80,90,122,137,168,219,230,232,245],"role":[42],"mechanisms":[45,60,206,246],"transformers,":[48],"their":[49],"effect":[50],"is":[51],"largely":[52],"unknown.":[53],"Here,":[54],"we":[55,78,149,192],"asked":[56],"if":[57,194],"transformers":[63,128,249],"exhibit":[64,197],"similar":[65,198],"effects":[66,139,199],"as":[67,140,177,185,200],"those":[68,141,201],"known":[69,142],"human":[71,113,212,236],"visual":[72,114,174,204,213],"attention.":[73,214,259],"To":[74,145],"answer":[75],"this":[76],"question,":[77],"revisited":[79],"formulation":[82],"and":[86,119,237,257],"found":[87,216],"despite":[89],"name,":[91],"computationally,":[92],"perform":[95,250],"special":[97],"class":[98],"relaxation":[100],"labeling":[101],"with":[102],"similarity":[103,172,256],"grouping":[104,151],"effects.":[105],"Additionally,":[106],"whereas":[107],"modern":[108],"experimental":[109],"findings":[110],"reveal":[111],"involves":[116],"both":[117,235],"feed-forward":[118,124,203],"feedback":[120],"mechanisms,":[121],"purely":[123],"architecture":[125],"suggests":[129,243],"cannot":[135],"have":[136],"same":[138],"humans.":[144],"quantify":[146],"observations,":[148],"evaluated":[150],"performance":[152],"family":[155],"Our":[159],"results":[160],"suggest":[161],"self-attention":[163],"modules":[164,222],"group":[165],"figures":[166],"stimuli":[169],"based":[170,253],"on":[171,254],"features":[175],"such":[176],"color.":[178],"Also,":[179],"singleton":[182],"detection":[183],"experiment":[184],"instance":[187],"salient":[189],"object":[190],"detection,":[191],"studied":[193],"salience":[205,225],"thought":[207],"be":[209],"utilized":[210],"We":[215],"generally,":[218],"transformer-based":[220],"assign":[223],"more":[224],"either":[226],"distractors":[228],"or":[229],"ground,":[231],"opposite":[233],"salience.":[239],"Together,":[240],"our":[241],"study":[242],"perceptual":[251],"organization":[252],"feature":[255],"not":[258]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}