{"id":"https://openalex.org/W3202467213","doi":"https://doi.org/10.48550/arxiv.2110.02095","title":"Exploring the Limits of Large Scale Pre-training","display_name":"Exploring the Limits of Large Scale Pre-training","alternate_titles":[],"publication_year":2021,"publication_date":"2021-10-05","ids":{"openalex":"https://openalex.org/W3202467213","doi":"https://doi.org/10.48550/arxiv.2110.02095","mag":"3202467213"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2110.02095","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.02095","pdf_url":"https://arxiv.org/pdf/2110.02095","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W3202467213"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2110.02095","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084377683","display_name":"Samira Abnar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abnar, Samira","raw_affiliation_strings":["Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102906366","display_name":"Mostafa Dehghani","orcid":"https://orcid.org/0000-0002-9772-1095","observed_orcids":["https://orcid.org/0000-0002-9772-1095"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dehghani, Mostafa","raw_affiliation_strings":["Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035369215","display_name":"Behnam Neyshabur","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Neyshabur, Behnam","raw_affiliation_strings":["Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045805879","display_name":"Hanie Sedghi","orcid":"https://orcid.org/0000-0002-7163-5009","observed_orcids":["https://orcid.org/0000-0002-7163-5009"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sedghi, Hanie","raw_affiliation_strings":["Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":134,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9437999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9437999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.05169999971985817,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.0020000000949949026,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-scale-pre-training","display_name":"large-scale pre-training","score":0.8600000143051147},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.7139999866485596},{"id":"https://openalex.org/keywords/performance-saturation","display_name":"performance saturation","score":0.7120000123977661},{"id":"https://openalex.org/keywords/downstream-tasks","display_name":"downstream tasks","score":0.625},{"id":"https://openalex.org/keywords/image-recognition","display_name":"image recognition","score":0.6370000243186951},{"id":"https://openalex.org/keywords/vision-transformer","display_name":"vision transformer","score":0.8949999809265137},{"id":"https://openalex.org/keywords/mlp-mixer","display_name":"MLP-Mixer","score":0.8510000109672546},{"id":"https://openalex.org/keywords/resnet","display_name":"ResNet","score":0.8460000157356262},{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.7089999914169312},{"id":"https://openalex.org/keywords/model-scaling","display_name":"model scaling","score":0.4729999899864197}],"concepts":[{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6919888257980347},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.6246140003204346},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.38426369428634644},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.24395713210105896},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.14970186352729797},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.08703777194023132}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2110.02095","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.02095","pdf_url":"https://arxiv.org/pdf/2110.02095","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2110.02095","doi":"https://doi.org/10.48550/arxiv.2110.02095","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2110.02095","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2110.02095","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.02095","pdf_url":"https://arxiv.org/pdf/2110.02095","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3202467213.pdf","grobid_xml":"https://content.openalex.org/works/W3202467213.grobid-xml"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W2047643928","https://openalex.org/W2108598243","https://openalex.org/W2115579991","https://openalex.org/W2117539524","https://openalex.org/W2134557905","https://openalex.org/W2138011018","https://openalex.org/W2149933564","https://openalex.org/W2155904486","https://openalex.org/W2561715562","https://openalex.org/W2592962403","https://openalex.org/W2626778328","https://openalex.org/W2709553318","https://openalex.org/W2799269579","https://openalex.org/W2804935296","https://openalex.org/W2806021783","https://openalex.org/W2893562797","https://openalex.org/W2901026139","https://openalex.org/W2919053476","https://openalex.org/W2945134764","https://openalex.org/W2946948417","https://openalex.org/W2947706148","https://openalex.org/W2953937638","https://openalex.org/W2962843773","https://openalex.org/W2963739978","https://openalex.org/W2964121744","https://openalex.org/W2964194231","https://openalex.org/W3001279689","https://openalex.org/W3008526508","https://openalex.org/W3015146382","https://openalex.org/W3030163527","https://openalex.org/W3035003500","https://openalex.org/W3092869576","https://openalex.org/W3094502228","https://openalex.org/W3098033339","https://openalex.org/W3104911444","https://openalex.org/W3112486745","https://openalex.org/W3126004537","https://openalex.org/W3128590981","https://openalex.org/W3129910129","https://openalex.org/W3131254173","https://openalex.org/W3134454711","https://openalex.org/W3135367836","https://openalex.org/W3135715136","https://openalex.org/W3135975972","https://openalex.org/W3143315506","https://openalex.org/W3147387781","https://openalex.org/W3156891177","https://openalex.org/W3157506437","https://openalex.org/W3169976301","https://openalex.org/W3172942063","https://openalex.org/W3175343838","https://openalex.org/W3180562345","https://openalex.org/W3199241049","https://openalex.org/W3212449366","https://openalex.org/W3214042613","https://openalex.org/W4394652010"],"related_works":["https://openalex.org/W4393905476","https://openalex.org/W7127915845","https://openalex.org/W4390190369","https://openalex.org/W4390963175","https://openalex.org/W7130409002","https://openalex.org/W7114781741","https://openalex.org/W3199241049","https://openalex.org/W4416034266","https://openalex.org/W3034993698","https://openalex.org/W7203681626"],"abstract_inverted_index":{"Recent":[0],"developments":[1],"in":[2,23,118],"large-scale":[3],"machine":[4],"learning":[5],"suggest":[6],"that":[7,21,108,131,139,151],"by":[8],"scaling":[9],"up":[10],"data,":[11],"model":[12,104],"size":[13],"and":[14,40,67,91,113,122,171],"training":[15],"time":[16],"properly,":[17],"one":[18],"might":[19],"observe":[20,144],"improvements":[22],"pre-training":[24],"would":[25],"transfer":[26],"favorably":[27],"to":[28,77,127,134,148,181,189],"most":[29],"downstream":[30,52,97,106,123,172,185],"tasks.":[31,100,124],"In":[32,55],"this":[33,38],"work,":[34],"we":[35,44,57,137,143,187],"systematically":[36],"study":[37],"phenomena":[39,112],"establish":[41],"that,":[42],"as":[43],"increase":[45],"the":[46,49,82,110,115,129,140,149,155,158],"upstream":[47,121,170,191],"accuracy,":[48],"performance":[50,107,119,168],"of":[51,71,85,120,157],"tasks":[53],"saturates.":[54],"particular,":[56],"investigate":[58],"more":[59,94,164],"than":[60,95],"4800":[61],"experiments":[62],"on":[63,81,93,169],"Vision":[64],"Transformers,":[65],"MLP-Mixers":[66],"ResNets":[68],"with":[69,176],"number":[70],"parameters":[72],"ranging":[73],"from":[74],"ten":[75,78],"million":[76],"billion,":[79],"trained":[80],"largest":[83],"scale":[84],"available":[86],"image":[87,98],"data":[88],"(JFT,":[89],"ImageNet21K)":[90],"evaluated":[92],"20":[96],"recognition":[99],"We":[101,160],"propose":[102],"a":[103,183],"for":[105],"reflects":[109],"saturation":[111,141],"captures":[114],"nonlinear":[116],"relationship":[117],"Delving":[125],"deeper":[126],"understand":[128],"reasons":[130],"give":[132],"rise":[133],"these":[135],"phenomena,":[136],"show":[138],"behavior":[142],"is":[145],"closely":[146],"related":[147],"way":[150],"representations":[152],"evolve":[153],"through":[154],"layers":[156],"models.":[159],"showcase":[161],"an":[162],"even":[163],"extreme":[165],"scenario":[166],"where":[167],"are":[173],"at":[174],"odds":[175],"each":[177],"other.":[178],"That":[179],"is,":[180],"have":[182],"better":[184],"performance,":[186],"need":[188],"hurt":[190],"accuracy.":[192]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3202467213","counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":18},{"year":2024,"cited_by_count":31},{"year":2023,"cited_by_count":38},{"year":2022,"cited_by_count":30},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2021-10-11T00:00:00"}
