{"id":"https://openalex.org/W3095872829","doi":"https://doi.org/10.48550/arxiv.2010.15327","title":"Do Wide and Deep Networks Learn the Same Things? Uncovering How Neural Network Representations Vary with Width and Depth","display_name":"Do Wide and Deep Networks Learn the Same Things? Uncovering How Neural Network Representations Vary with Width and Depth","publication_year":2020,"publication_date":"2020-10-29","ids":{"openalex":"https://openalex.org/W3095872829","doi":"https://doi.org/10.48550/arxiv.2010.15327","mag":"3095872829"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2010.15327","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2010.15327","pdf_url":"https://arxiv.org/pdf/2010.15327","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2010.15327","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102926742","display_name":"Thao Ngoc Nguyen","orcid":"https://orcid.org/0000-0002-5016-2230","observed_orcids":["https://orcid.org/0000-0002-5016-2230"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nguyen, Thao","raw_affiliation_strings":["Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027041171","display_name":"Maithra Raghu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Raghu, Maithra","raw_affiliation_strings":["Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042760897","display_name":"Simon Kornblith","orcid":"https://orcid.org/0000-0002-9088-2443","observed_orcids":["https://orcid.org/0000-0002-9088-2443"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kornblith, Simon","raw_affiliation_strings":["Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":134,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6685000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6685000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.12269999831914902,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.09969999641180038,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hidden-representations","display_name":"hidden representations","score":0.6430000066757202},{"id":"https://openalex.org/keywords/block-structure","display_name":"block structure","score":0.7829999923706055},{"id":"https://openalex.org/keywords/model-capacity","display_name":"model capacity","score":0.8569999933242798},{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.45500001311302185},{"id":"https://openalex.org/keywords/error-patterns","display_name":"error patterns","score":0.6259999871253967},{"id":"https://openalex.org/keywords/model-architecture-comparison","display_name":"model architecture comparison","score":0.42899999022483826},{"id":"https://openalex.org/keywords/overfitting","display_name":"overfitting","score":0.5910000205039978}],"concepts":[{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.6910809278488159},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6367383599281311},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.574763298034668},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5441729426383972},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5091354250907898},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5018565654754639},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.4997529983520508},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.48603278398513794},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4703787863254547},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4610099196434021},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4570889174938202},{"id":"https://openalex.org/C193415008","wikidata":"https://www.wikidata.org/wiki/Q639681","display_name":"Network architecture","level":2,"score":0.4324411451816559},{"id":"https://openalex.org/C104122410","wikidata":"https://www.wikidata.org/wiki/Q1416406","display_name":"Network model","level":2,"score":0.41494283080101013},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.34203821420669556},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22707942128181458},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.1157887876033783},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.07410770654678345},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2010.15327","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2010.15327","pdf_url":"https://arxiv.org/pdf/2010.15327","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2010.15327","doi":"https://doi.org/10.48550/arxiv.2010.15327","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2010.15327","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:3127775241","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2010.15327","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2010.15327","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2010.15327","pdf_url":"https://arxiv.org/pdf/2010.15327","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3095872829.pdf","grobid_xml":"https://content.openalex.org/works/W3095872829.grobid-xml"},"referenced_works_count":45,"referenced_works":["https://openalex.org/W1690739335","https://openalex.org/W1988115241","https://openalex.org/W2028520303","https://openalex.org/W2103496339","https://openalex.org/W2108598243","https://openalex.org/W2127658298","https://openalex.org/W2158581396","https://openalex.org/W2194775991","https://openalex.org/W2208555118","https://openalex.org/W2433379750","https://openalex.org/W2537199094","https://openalex.org/W2766099245","https://openalex.org/W2766678531","https://openalex.org/W2767204723","https://openalex.org/W2785626633","https://openalex.org/W2809090039","https://openalex.org/W2810898455","https://openalex.org/W2899032424","https://openalex.org/W2899402383","https://openalex.org/W2910655610","https://openalex.org/W2942810103","https://openalex.org/W2946948417","https://openalex.org/W2952204734","https://openalex.org/W2952502547","https://openalex.org/W2963097630","https://openalex.org/W2963569887","https://openalex.org/W2963759780","https://openalex.org/W2964137095","https://openalex.org/W2970373599","https://openalex.org/W2970854840","https://openalex.org/W2971044152","https://openalex.org/W2971293928","https://openalex.org/W2972249811","https://openalex.org/W2989457543","https://openalex.org/W2989606769","https://openalex.org/W2989609934","https://openalex.org/W3003414992","https://openalex.org/W3009686669","https://openalex.org/W3022561973","https://openalex.org/W3035354904","https://openalex.org/W3035547806","https://openalex.org/W3036770516","https://openalex.org/W3098033339","https://openalex.org/W3100924069","https://openalex.org/W3118608800"],"related_works":["https://openalex.org/W2033337358","https://openalex.org/W3005526090","https://openalex.org/W4226052297","https://openalex.org/W7202335387","https://openalex.org/W2182736213","https://openalex.org/W7222669498","https://openalex.org/W6885174648","https://openalex.org/W7133224629","https://openalex.org/W3166609963","https://openalex.org/W6967018793"],"abstract_inverted_index":{"A":[0],"key":[1],"factor":[2],"in":[3,35,85],"the":[4,11,21,56,86,110,113,120,126,147,161,171,182],"success":[5],"of":[6,29,42,49,51,89,112,119,130,174],"deep":[7,189],"neural":[8,30],"networks":[9],"is":[10,46,106,117,164,185],"ability":[12],"to":[13,16,109,166],"scale":[14],"models":[15,190],"improve":[17],"performance":[18],"by":[19,69,141],"varying":[20,72,156],"architecture":[22],"depth":[23,52,73],"and":[24,53,74,116,124,158,188,195],"width.":[25],"This":[26,133],"simple":[27],"property":[28],"network":[31],"design":[32],"has":[33,135],"resulted":[34],"highly":[36],"effective":[37],"architectures":[38,154],"for":[39,138],"a":[40,81],"variety":[41],"tasks.":[43],"Nevertheless,":[44],"there":[45],"limited":[47],"understanding":[48],"effects":[50],"width":[54,75],"on":[55],"learned":[57,140],"representations.":[58,132],"In":[59],"this":[60,64,99],"paper,":[61],"we":[62],"study":[63],"fundamental":[65],"question.":[66],"We":[67,96,169],"begin":[68],"investigating":[70],"how":[71],"affects":[76],"model":[77,104,176],"hidden":[78,87],"representations,":[79],"finding":[80,178],"characteristic":[82],"block":[83,100,148,162],"structure":[84,101,149,163],"representations":[88,145],"larger":[90],"capacity":[91,105],"(wider":[92],"or":[93],"deeper)":[94],"models.":[95],"demonstrate":[97],"that":[98,179],"arises":[102],"when":[103,181],"large":[107],"relative":[108],"size":[111],"training":[114],"set,":[115],"indicative":[118],"underlying":[121],"layers":[122],"preserving":[123],"propagating":[125],"dominant":[127],"principal":[128],"component":[129],"their":[131],"discovery":[134],"important":[136],"ramifications":[137],"features":[139],"different":[142,175],"models,":[143],"namely,":[144],"outside":[146],"are":[150],"often":[151],"similar":[152],"across":[153,197],"with":[155],"widths":[157],"depths,":[159],"but":[160],"unique":[165],"each":[167],"model.":[168],"analyze":[170],"output":[172],"predictions":[173],"architectures,":[177],"even":[180],"overall":[183],"accuracy":[184],"similar,":[186],"wide":[187],"exhibit":[191],"distinctive":[192],"error":[193],"patterns":[194],"variations":[196],"classes.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":28},{"year":2023,"cited_by_count":39},{"year":2022,"cited_by_count":21},{"year":2021,"cited_by_count":21},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2020-11-09T00:00:00"}