{"id":"https://openalex.org/W3137182587","doi":"https://doi.org/10.1609/aaai.v36i2.20094","title":"Can Vision Transformers Learn without Natural Images?","display_name":"Can Vision Transformers Learn without Natural Images?","publication_year":2022,"publication_date":"2022-06-28","ids":{"openalex":"https://openalex.org/W3137182587","doi":"https://doi.org/10.1609/aaai.v36i2.20094","mag":"3137182587"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v36i2.20094","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v36i2.20094","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v36i2.20094","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009991570","display_name":"Kodai Nakashima","orcid":"https://orcid.org/0000-0003-1539-3567"},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kodai Nakashima","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST)","institution_ids":["https://openalex.org/I73613424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011507481","display_name":"Hirokatsu Kataoka","orcid":"https://orcid.org/0000-0001-8844-165X"},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hirokatsu Kataoka","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST)","institution_ids":["https://openalex.org/I73613424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083532594","display_name":"Asato Matsumoto","orcid":null},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Asato Matsumoto","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST)","institution_ids":["https://openalex.org/I73613424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060846465","display_name":"Kenji Iwata","orcid":"https://orcid.org/0000-0001-6609-8221"},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kenji Iwata","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST)","institution_ids":["https://openalex.org/I73613424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101261291","display_name":"Nakamasa Inoue","orcid":"https://orcid.org/0009-0006-1145-0914"},"institutions":[{"id":"https://openalex.org/I114531698","display_name":"Tokyo Institute of Technology","ror":"https://ror.org/0112mx960","country_code":"JP","type":"education","lineage":["https://openalex.org/I114531698"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Nakamasa Inoue","raw_affiliation_strings":["Tokyo Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tokyo Institute of Technology","institution_ids":["https://openalex.org/I114531698"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043995369","display_name":"Yutaka Satoh","orcid":"https://orcid.org/0000-0002-0638-0855"},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yutaka Satoh","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST)","institution_ids":["https://openalex.org/I73613424"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.759,"has_fulltext":true,"cited_by_count":34,"citation_normalized_percentile":{"value":0.93510408,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"36","issue":"2","first_page":"1990","last_page":"1998"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7630876302719116},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6764514446258545},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6699209213256836},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.5550925731658936},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.5305331945419312},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.46658116579055786},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.46070021390914917},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.39084723591804504},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.06886601448059082}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7630876302719116},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6764514446258545},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6699209213256836},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.5550925731658936},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.5305331945419312},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46658116579055786},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.46070021390914917},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.39084723591804504},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.06886601448059082},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1609/aaai.v36i2.20094","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v36i2.20094","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/20094","is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/20094","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/20094/19853","source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:t2r2.star.titech.ac.jp:50590855","is_oa":false,"landing_page_url":"http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100859047","pdf_url":null,"source":{"id":"https://openalex.org/S4377196385","display_name":"Tokyo Tech Research Repository (Tokyo Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I114531698","host_organization_name":"Tokyo Institute of Technology","host_organization_lineage":["https://openalex.org/I114531698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"}],"best_oa_location":{"id":"doi:10.1609/aaai.v36i2.20094","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v36i2.20094","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth","score":0.7099999785423279}],"awards":[{"id":"https://openalex.org/G3171956239","display_name":"Deep Learning Principle Analysis in Accordance with Natural Law","funder_award_id":"19H01134","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G3187968308","display_name":null,"funder_award_id":"JPNP 20006","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G8893416511","display_name":null,"funder_award_id":"JPNP20006","funder_id":"https://openalex.org/F4320311508","funder_display_name":"National Institute of Advanced Industrial Science and Technology"}],"funders":[{"id":"https://openalex.org/F4320311508","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54"},{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":65,"referenced_works":["https://openalex.org/W343636949","https://openalex.org/W1686810756","https://openalex.org/W1849277567","https://openalex.org/W2078206416","https://openalex.org/W2097117768","https://openalex.org/W2108598243","https://openalex.org/W2117539524","https://openalex.org/W2145607950","https://openalex.org/W2163605009","https://openalex.org/W2194775991","https://openalex.org/W2321533354","https://openalex.org/W2326925005","https://openalex.org/W2339378878","https://openalex.org/W2549139847","https://openalex.org/W2626778328","https://openalex.org/W2732026016","https://openalex.org/W2734663976","https://openalex.org/W2750549109","https://openalex.org/W2785325870","https://openalex.org/W2799113232","https://openalex.org/W2800860906","https://openalex.org/W2896457183","https://openalex.org/W2946948417","https://openalex.org/W2950187998","https://openalex.org/W2955425717","https://openalex.org/W2962835968","https://openalex.org/W2962843773","https://openalex.org/W2963341956","https://openalex.org/W2963446712","https://openalex.org/W2963465221","https://openalex.org/W2963703197","https://openalex.org/W2987283559","https://openalex.org/W2991391304","https://openalex.org/W2996535895","https://openalex.org/W3001279689","https://openalex.org/W3005680577","https://openalex.org/W3009561768","https://openalex.org/W3014034447","https://openalex.org/W3034978746","https://openalex.org/W3035524453","https://openalex.org/W3036224891","https://openalex.org/W3036982689","https://openalex.org/W3094502228","https://openalex.org/W3097217077","https://openalex.org/W3098903812","https://openalex.org/W3100859887","https://openalex.org/W3101298150","https://openalex.org/W3112788634","https://openalex.org/W3116489684","https://openalex.org/W3119786062","https://openalex.org/W3122542623","https://openalex.org/W3170874841","https://openalex.org/W4292779060","https://openalex.org/W4385245566","https://openalex.org/W4394645241","https://openalex.org/W6674914833","https://openalex.org/W6676297131","https://openalex.org/W6687483927","https://openalex.org/W6701655646","https://openalex.org/W6703787047","https://openalex.org/W6739901393","https://openalex.org/W6743135768","https://openalex.org/W6756427901","https://openalex.org/W6766120893","https://openalex.org/W6787774617"],"related_works":["https://openalex.org/W4283262748","https://openalex.org/W2961085424","https://openalex.org/W2502773048","https://openalex.org/W1979668694","https://openalex.org/W48866389","https://openalex.org/W4224009465","https://openalex.org/W2117651364","https://openalex.org/W3112533414","https://openalex.org/W4306674287","https://openalex.org/W2490481324"],"abstract_inverted_index":{"Is":[0],"it":[1],"possible":[2],"to":[3,30,54,111,165,190],"complete":[4],"Vision":[5],"Transformer":[6],"(ViT)":[7],"pre-training":[8,28,105],"without":[9,98,145],"natural":[10,38,47,101,146,162],"images":[11,39,48,102,147],"and":[12,40,60,84,96,198],"human-annotated":[13,41],"labels?":[14],"This":[15],"question":[16],"has":[17,49],"become":[18],"increasingly":[19],"relevant":[20],"in":[21,51,103,127,170],"recent":[22,44],"months":[23],"because":[24],"while":[25,142],"current":[26],"ViT":[27],"tends":[29],"rely":[31],"heavily":[32],"on":[33,116],"a":[34,166,182],"large":[35,167],"number":[36],"of":[37,46,75,135,185,193],"labels,":[42],"the":[43,61,73,104,128,173,191],"use":[45],"resulted":[50],"problems":[52],"related":[53],"privacy":[55],"violation,":[56],"inadequate":[57],"fairness":[58],"protection,":[59],"need":[62],"for":[63,124],"labor-intensive":[64],"annotations.":[65],"In":[66],"this":[67],"paper,":[68],"we":[69,139,176],"experimentally":[70],"verify":[71],"that":[72,120,141,150,178],"results":[74],"formula-driven":[76],"supervised":[77],"learning":[78,91],"(FDSL)":[79],"framework":[80],"are":[81,151],"comparable":[82,189],"with,":[83],"can":[85,159],"even":[86],"partially":[87],"outperform,":[88],"sophisticated":[89],"self-supervised":[90],"(SSL)":[92],"methods":[93],"like":[94],"SimCLRv2":[95,197],"MoCov2":[97],"using":[99,172],"any":[100],"phase.":[106],"We":[107],"also":[108],"consider":[109],"ways":[110],"reorganize":[112],"FractalDB":[113],"generation":[114],"based":[115],"our":[117,179],"tentative":[118],"conclusion":[119],"there":[121],"is":[122,188],"room":[123],"configuration":[125],"improvements":[126],"iterated":[129],"function":[130],"system":[131],"(IFS)":[132],"parameter":[133],"settings":[134],"such":[136],"databases.":[137],"Moreover,":[138],"show":[140,177],"ViTs":[143],"pre-trained":[144,156],"produce":[148],"visualizations":[149],"somewhat":[152],"different":[153],"from":[154],"ImageNet":[155],"ViTs,":[157],"they":[158],"still":[160],"interpret":[161],"image":[163],"datasets":[164],"extent.":[168],"Finally,":[169],"experiments":[171],"CIFAR-10":[174],"dataset,":[175],"model":[180],"achieved":[181,195,200],"performance":[183],"rate":[184,192],"97.8,":[186],"which":[187],"97.4":[194],"with":[196,201],"98.0":[199],"ImageNet.":[202]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":3}],"updated_date":"2026-08-25T07:29:55.448023","created_date":"2025-10-10T00:00:00"}