{"id":"https://openalex.org/W4415056349","doi":"https://doi.org/10.1007/978-3-032-08317-3_2","title":"From Colors to\u00a0Classes: Emergence of\u00a0Concepts in\u00a0Vision Transformers","display_name":"From Colors to\u00a0Classes: Emergence of\u00a0Concepts in\u00a0Vision Transformers","publication_year":2025,"publication_date":"2025-10-11","ids":{"openalex":"https://openalex.org/W4415056349","doi":"https://doi.org/10.1007/978-3-032-08317-3_2"},"language":"en","primary_location":{"id":"doi:10.1007/978-3-032-08317-3_2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-08317-3_2","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08317-3_2.pdf","source":{"id":"https://openalex.org/S2764900261","display_name":"Communications in computer and information science","issn_l":"1865-0929","issn":["1865-0929","1865-0937"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications in Computer and Information Science","raw_type":"book-chapter"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08317-3_2.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092050270","display_name":"Teresa Karen Scheidt","orcid":"https://orcid.org/0009-0002-0283-2508"},"institutions":[{"id":"https://openalex.org/I96673099","display_name":"Technical University of Denmark","ror":"https://ror.org/04qtj9h94","country_code":"DK","type":"education","lineage":["https://openalex.org/I96673099"]}],"countries":["DK"],"is_corresponding":true,"raw_author_name":"Teresa Dorszewski","raw_affiliation_strings":["Department of Applied Mathematics and Computer Science, Technical University of Denmark, Kongens Lyngby, Denmark"],"raw_orcid":"https://orcid.org/0009-0002-0283-2508","affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Computer Science, Technical University of Denmark, Kongens Lyngby, Denmark","institution_ids":["https://openalex.org/I96673099"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055051292","display_name":"Lenka T\u011btkov\u00e1","orcid":"https://orcid.org/0000-0002-0009-6896"},"institutions":[{"id":"https://openalex.org/I96673099","display_name":"Technical University of Denmark","ror":"https://ror.org/04qtj9h94","country_code":"DK","type":"education","lineage":["https://openalex.org/I96673099"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Lenka T\u011btkov\u00e1","raw_affiliation_strings":["Department of Applied Mathematics and Computer Science, Technical University of Denmark, Kongens Lyngby, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-0009-6896","affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Computer Science, Technical University of Denmark, Kongens Lyngby, Denmark","institution_ids":["https://openalex.org/I96673099"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013414039","display_name":"Robert Jenssen","orcid":"https://orcid.org/0000-0002-7496-8474"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]},{"id":"https://openalex.org/I139025015","display_name":"Pioneer (United States)","ror":"https://ror.org/05jke7315","country_code":"US","type":"company","lineage":["https://openalex.org/I139025015","https://openalex.org/I4210104019"]},{"id":"https://openalex.org/I144648426","display_name":"Norwegian Computing Center","ror":"https://ror.org/02gm7te43","country_code":"NO","type":"nonprofit","lineage":["https://openalex.org/I144648426"]},{"id":"https://openalex.org/I78037679","display_name":"UiT The Arctic University of Norway","ror":"https://ror.org/00wge5k78","country_code":"NO","type":"education","lineage":["https://openalex.org/I78037679"]}],"countries":["DK","NO","US"],"is_corresponding":false,"raw_author_name":"Robert Jenssen","raw_affiliation_strings":["Department of Physics and Technology, UiT The Arctic University of Norway, Troms\u00f8, Norway","Norwegian Computing Center, Oslo, Norway","Pioneer Centre for AI, University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-7496-8474","affiliations":[{"raw_affiliation_string":"Department of Physics and Technology, UiT The Arctic University of Norway, Troms\u00f8, Norway","institution_ids":["https://openalex.org/I78037679"]},{"raw_affiliation_string":"Norwegian Computing Center, Oslo, Norway","institution_ids":["https://openalex.org/I144648426"]},{"raw_affiliation_string":"Pioneer Centre for AI, University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696","https://openalex.org/I139025015"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018292103","display_name":"Lars Kai Hansen","orcid":"https://orcid.org/0000-0003-0442-5877"},"institutions":[{"id":"https://openalex.org/I96673099","display_name":"Technical University of Denmark","ror":"https://ror.org/04qtj9h94","country_code":"DK","type":"education","lineage":["https://openalex.org/I96673099"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Lars Kai Hansen","raw_affiliation_strings":["Department of Applied Mathematics and Computer Science, Technical University of Denmark, Kongens Lyngby, Denmark"],"raw_orcid":"https://orcid.org/0000-0003-0442-5877","affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Computer Science, Technical University of Denmark, Kongens Lyngby, Denmark","institution_ids":["https://openalex.org/I96673099"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031748191","display_name":"Kristoffer Wickstr\u00f8m","orcid":"https://orcid.org/0000-0003-1395-7154"},"institutions":[{"id":"https://openalex.org/I78037679","display_name":"UiT The Arctic University of Norway","ror":"https://ror.org/00wge5k78","country_code":"NO","type":"education","lineage":["https://openalex.org/I78037679"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Kristoffer Knutsen Wickstr\u00f8m","raw_affiliation_strings":["Department of Physics and Technology, UiT The Arctic University of Norway, Troms\u00f8, Norway"],"raw_orcid":"https://orcid.org/0000-0003-1395-7154","affiliations":[{"raw_affiliation_string":"Department of Physics and Technology, UiT The Arctic University of Norway, Troms\u00f8, Norway","institution_ids":["https://openalex.org/I78037679"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5092050270"],"corresponding_institution_ids":["https://openalex.org/I96673099"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"28","last_page":"47"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9358000159263611,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9358000159263611,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14319","display_name":"Currency Recognition and Detection","score":0.9266999959945679,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/encode","display_name":"ENCODE","score":0.8608999848365784},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6299999952316284},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.5153999924659729},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4699000120162964},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.462799996137619},{"id":"https://openalex.org/keywords/domain-adaptation","display_name":"Domain adaptation","score":0.4278999865055084},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4223000109195709},{"id":"https://openalex.org/keywords/adaptation","display_name":"Adaptation (eye)","score":0.41589999198913574}],"concepts":[{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.8608999848365784},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7864999771118164},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6299999952316284},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.5153999924659729},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48899999260902405},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4699000120162964},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.462799996137619},{"id":"https://openalex.org/C2776434776","wikidata":"https://www.wikidata.org/wiki/Q19246213","display_name":"Domain adaptation","level":3,"score":0.4278999865055084},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4223000109195709},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.41589999198913574},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.39959999918937683},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.38339999318122864},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3343999981880188},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.32190001010894775},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.32120001316070557},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.2856000065803528},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.2754000127315521},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.265500009059906},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.26170000433921814},{"id":"https://openalex.org/C87868495","wikidata":"https://www.wikidata.org/wiki/Q750843","display_name":"Information processing","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C8854915","wikidata":"https://www.wikidata.org/wiki/Q4350200","display_name":"Information transfer","level":2,"score":0.2583000063896179},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25130000710487366}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/978-3-032-08317-3_2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-08317-3_2","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08317-3_2.pdf","source":{"id":"https://openalex.org/S2764900261","display_name":"Communications in computer and information science","issn_l":"1865-0929","issn":["1865-0929","1865-0937"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications in Computer and Information Science","raw_type":"book-chapter"}],"best_oa_location":{"id":"doi:10.1007/978-3-032-08317-3_2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-08317-3_2","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08317-3_2.pdf","source":{"id":"https://openalex.org/S2764900261","display_name":"Communications in computer and information science","issn_l":"1865-0929","issn":["1865-0929","1865-0937"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications in Computer and Information Science","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1755387187","display_name":null,"funder_award_id":"NNF22OC0076907","funder_id":"https://openalex.org/F4320325957","funder_display_name":"Novo Nordisk Fonden"},{"id":"https://openalex.org/G6800035256","display_name":null,"funder_award_id":"309439","funder_id":"https://openalex.org/F4320323299","funder_display_name":"Norges Forskningsr\u00e5d"},{"id":"https://openalex.org/G8281805605","display_name":null,"funder_award_id":"303514","funder_id":"https://openalex.org/F4320323299","funder_display_name":"Norges Forskningsr\u00e5d"}],"funders":[{"id":"https://openalex.org/F4320320943","display_name":"Danmarks Grundforskningsfond","ror":"https://ror.org/00znyv691"},{"id":"https://openalex.org/F4320322436","display_name":"Novo Nordisk","ror":"https://ror.org/0435rc536"},{"id":"https://openalex.org/F4320323299","display_name":"Norges Forskningsr\u00e5d","ror":"https://ror.org/00epmv149"},{"id":"https://openalex.org/F4320325957","display_name":"Novo Nordisk Fonden","ror":"https://ror.org/04txyc737"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415056349.pdf","grobid_xml":"https://content.openalex.org/works/W4415056349.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1849277567","https://openalex.org/W2117539524","https://openalex.org/W2178314882","https://openalex.org/W2194775991","https://openalex.org/W2560647685","https://openalex.org/W2792641098","https://openalex.org/W2963749936","https://openalex.org/W2979826702","https://openalex.org/W3010694149","https://openalex.org/W3015930014","https://openalex.org/W3016970897","https://openalex.org/W3103934428","https://openalex.org/W4213019189","https://openalex.org/W4297812995","https://openalex.org/W4301785137","https://openalex.org/W4312364648","https://openalex.org/W4313156423","https://openalex.org/W4317436377","https://openalex.org/W4366208220","https://openalex.org/W4386076059","https://openalex.org/W4405254344"],"related_works":[],"abstract_inverted_index":{"Abstract":[0],"Vision":[1],"Transformers":[2],"(ViTs)":[3],"are":[4],"increasingly":[5],"utilized":[6],"in":[7,85,111,164],"various":[8],"computer":[9,86],"vision":[10],"tasks":[11,50,195],"due":[12,74],"to":[13,75,92,192,207],"their":[14,44,76],"powerful":[15],"representation":[16],"capabilities.":[17],"However,":[18],"it":[19],"remains":[20],"understudied":[21],"how":[22],"ViTs":[23,84,113,121],"process":[24],"information":[25],"layer":[26,166],"by":[27],"layer.":[28],"Numerous":[29],"studies":[30],"have":[31],"shown":[32],"that":[33,120],"convolutional":[34],"neural":[35],"networks":[36],"(CNNs)":[37],"extract":[38],"features":[39,135],"of":[40,83,97,108,155,161,176,187,200],"increasing":[41,81,125],"complexity":[42,126,154],"throughout":[43,127],"layers,":[45],"which":[46],"is":[47,89],"crucial":[48],"for":[49],"like":[51],"domain":[52],"adaptation":[53],"and":[54,139,150,173,185,203],"transfer":[55],"learning.":[56],"ViTs,":[57],"lacking":[58],"the":[59,71,80,94,128,153,159,183,198,205],"same":[60],"inductive":[61],"biases":[62],"as":[63,137],"CNNs,":[64],"can":[65],"potentially":[66],"learn":[67],"global":[68],"dependencies":[69],"from":[70],"first":[72],"layers":[73,131,143],"attention":[77],"mechanisms.":[78],"Given":[79],"importance":[82],"vision,":[87],"there":[88],"a":[90,104,170],"need":[91],"improve":[93],"layer-wise":[95,106],"understanding":[96],"ViTs.":[98],"In":[99],"this":[100],"work,":[101],"we":[102],"present":[103],"novel,":[105],"analysis":[107],"concepts":[109,123,157,162,202,206],"encoded":[110,156,188,201],"state-of-the-art":[112],"using":[114],"neuron":[115],"labeling.":[116],"Our":[117],"findings":[118],"reveal":[119],"encode":[122,133],"with":[124,190],"network.":[129],"Early":[130],"primarily":[132],"basic":[134],"such":[136],"colors":[138],"textures,":[140],"while":[141],"later":[142],"represent":[144],"more":[145,171,208],"specific":[146,174,193],"classes,":[147],"including":[148],"objects":[149],"animals.":[151],"As":[152],"increases,":[158],"number":[160,199],"represented":[163],"each":[165],"also":[167],"rises,":[168],"reflecting":[169],"diverse":[172],"set":[175],"features.":[177],"Additionally,":[178],"different":[179],"pretraining":[180],"strategies":[181],"influence":[182],"quantity":[184],"category":[186],"concepts,":[189],"finetuning":[191],"downstream":[194],"generally":[196],"reducing":[197],"shifting":[204],"relevant":[209],"categories.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-11T00:00:00"}