{"id":"https://openalex.org/W4221156853","doi":"https://doi.org/10.48550/arxiv.2203.08130","title":"One Network Doesn't Rule Them All: Moving Beyond Handcrafted Architectures in Self-Supervised Learning","display_name":"One Network Doesn't Rule Them All: Moving Beyond Handcrafted Architectures in Self-Supervised Learning","publication_year":2022,"publication_date":"2022-03-15","ids":{"openalex":"https://openalex.org/W4221156853","doi":"https://doi.org/10.48550/arxiv.2203.08130"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2203.08130","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2203.08130","pdf_url":"https://arxiv.org/pdf/2203.08130","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2203.08130","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033227351","display_name":"Sharath Girish","orcid":"https://orcid.org/0000-0003-4364-0262","observed_orcids":["https://orcid.org/0000-0003-4364-0262"]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Girish, Sharath","raw_affiliation_strings":["University of Maryland , College Park"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland , College Park","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dey, Debadeepta","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dey, Debadeepta","raw_affiliation_strings":["Microsoft Research , Redmond"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research , Redmond","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072006037","display_name":"Neel Joshi","orcid":"https://orcid.org/0000-0001-8236-3566","observed_orcids":["https://orcid.org/0000-0001-8236-3566"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joshi, Neel","raw_affiliation_strings":["Microsoft Research , Redmond"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research , Redmond","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045147286","display_name":"Vibhav Vineet","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vineet, Vibhav","raw_affiliation_strings":["Microsoft Research , Redmond"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research , Redmond","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shah, Shital","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shah, Shital","raw_affiliation_strings":["Microsoft Research , Redmond"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research , Redmond","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Mendes, Caio Cesar Teodoro","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mendes, Caio Cesar Teodoro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Shrivastava, Abhinav","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shrivastava, Abhinav","raw_affiliation_strings":["University of Maryland , College Park"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland , College Park","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"last","author":{"id":null,"display_name":"Song, Yale","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Song, Yale","raw_affiliation_strings":["Microsoft Research , Redmond"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research , Redmond","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.8773999810218811,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.8773999810218811,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0925000011920929,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.020500000566244125,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.9990000128746033},{"id":"https://openalex.org/keywords/network-architecture","display_name":"network architecture","score":0.5849999785423279},{"id":"https://openalex.org/keywords/architecture-search","display_name":"architecture search","score":0.6129999756813049},{"id":"https://openalex.org/keywords/resnet","display_name":"ResNet","score":0.7089999914169312},{"id":"https://openalex.org/keywords/mobilenet","display_name":"MobileNet","score":1.0},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.6949999928474426},{"id":"https://openalex.org/keywords/unlabeled-data","display_name":"unlabeled data","score":0.7799999713897705}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7784608602523804},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6937461495399475},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.6340008974075317},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.618384838104248},{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.5404331088066101},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.532730221748352},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.512752890586853},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.46891841292381287},{"id":"https://openalex.org/C193415008","wikidata":"https://www.wikidata.org/wiki/Q639681","display_name":"Network architecture","level":2,"score":0.46087366342544556},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4497607946395874},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.41316795349121094},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2203.08130","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2203.08130","pdf_url":"https://arxiv.org/pdf/2203.08130","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2203.08130","doi":"https://doi.org/10.48550/arxiv.2203.08130","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2203.08130","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2203.08130","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2203.08130","pdf_url":"https://arxiv.org/pdf/2203.08130","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3038994994","https://openalex.org/W4280500786","https://openalex.org/W3206305878","https://openalex.org/W7024685605","https://openalex.org/W4226105882","https://openalex.org/W4318751314","https://openalex.org/W4389713642","https://openalex.org/W4378768563","https://openalex.org/W4405466096","https://openalex.org/W7213981014"],"abstract_inverted_index":{"The":[0,21],"current":[1],"literature":[2],"on":[3,8,18,32,42,129,169],"self-supervised":[4,201],"learning":[5,10,67,202],"(SSL)":[6],"focuses":[7],"developing":[9],"objectives":[11,41],"to":[12,37,61,133,185],"train":[13],"neural":[14],"networks":[15],"more":[16],"effectively":[17],"unlabeled":[19],"data.":[20],"typical":[22],"development":[23],"process":[24],"involves":[25],"taking":[26],"well-established":[27],"architectures,":[28],"e.g.,":[29],"ResNet":[30,98],"demonstrated":[31],"ImageNet,":[33],"and":[34,99,102,157,165,176,193],"using":[35],"them":[36,104],"evaluate":[38,103],"newly":[39],"developed":[40],"downstream":[43,107],"scenarios.":[44,127],"While":[45],"convenient,":[46],"this":[47,70],"does":[48],"not":[49,135],"take":[50],"into":[51,200],"account":[52],"the":[53,65,110,126,144,163],"role":[54,85],"of":[55,97],"architectures":[56,101,155,190],"which":[57],"has":[58],"been":[59],"shown":[60],"be":[62],"crucial":[63],"in":[64,86,109,143,191],"supervised":[66],"literature.":[68],"In":[69],"work,":[71],"we":[72,131],"establish":[73],"extensive":[74],"empirical":[75],"evidence":[76],"showing":[77],"that":[78,115,121,149,181],"a":[79,83,90],"network":[80,120,137],"architecture":[81,141,198],"plays":[82],"significant":[84],"SSL.":[87],"We":[88,113,147],"conduct":[89],"large-scale":[91],"study":[92],"with":[93,162],"over":[94],"100":[95],"variants":[96],"MobileNet":[100],"across":[105,125],"11":[106],"scenarios":[108],"SSL":[111,145,192],"setting.":[112],"show":[114,148],"there":[116],"is":[117,183],"no":[118],"one":[119],"performs":[122],"consistently":[123],"well":[124],"Based":[128],"this,":[130],"propose":[132],"learn":[134],"only":[136],"weights":[138],"but":[139],"also":[140],"topologies":[142],"regime.":[146],"\"self-supervised":[150],"architectures\"":[151],"outperform":[152],"popular":[153],"handcrafted":[154,189],"(ResNet18":[156],"MobileNetV2)":[158],"while":[159],"performing":[160],"competitively":[161],"larger":[164],"computationally":[166],"heavy":[167],"ResNet50":[168],"major":[170],"image":[171],"classification":[172],"benchmarks":[173],"(ImageNet-1K,":[174],"iNat2021,":[175],"more).":[177],"Our":[178],"results":[179],"suggest":[180],"it":[182],"time":[184],"consider":[186],"moving":[187],"beyond":[188],"start":[194],"thinking":[195],"about":[196],"incorporating":[197],"search":[199],"objectives.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-17T09:40:24.721904","created_date":"2022-04-03T00:00:00"}