{"id":"https://openalex.org/W2933878393","doi":"https://doi.org/10.48550/arxiv.2003.06254","title":"What Information Does a ResNet Compress?","display_name":"What Information Does a ResNet Compress?","publication_year":2020,"publication_date":"2020-03-13","ids":{"openalex":"https://openalex.org/W2933878393","doi":"https://doi.org/10.48550/arxiv.2003.06254","mag":"2933878393"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2003.06254","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.06254","pdf_url":"https://arxiv.org/pdf/2003.06254","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2003.06254","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005605477","display_name":"Luke Nicholas Darlow","orcid":"https://orcid.org/0009-0009-6163-6807","observed_orcids":["https://orcid.org/0009-0009-6163-6807"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Darlow, Luke Nicholas","raw_affiliation_strings":["School of Informatics University of Edinburgh 10 Crichton St , Edinburgh EH8 9AB","University of Edinburgh,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Informatics University of Edinburgh 10 Crichton St , Edinburgh EH8 9AB","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"University of Edinburgh,","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007901825","display_name":"Amos Storkey","orcid":"https://orcid.org/0000-0002-8100-506X","observed_orcids":["https://orcid.org/0000-0002-8100-506X"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Storkey, Amos","raw_affiliation_strings":["School of Informatics University of Edinburgh 10 Crichton St , Edinburgh EH8 9AB","University of Edinburgh,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Informatics University of Edinburgh 10 Crichton St , Edinburgh EH8 9AB","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"University of Edinburgh,","institution_ids":["https://openalex.org/I98677209"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I98677209"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5439000129699707,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5439000129699707,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.19059999287128448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.11819999665021896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/information-bottleneck-principle","display_name":"information bottleneck theory","score":0.9559999704360962},{"id":"https://openalex.org/keywords/resnet","display_name":"ResNet","score":0.8050000071525574},{"id":"https://openalex.org/keywords/mutual-information","display_name":"mutual information","score":0.8790000081062317},{"id":"https://openalex.org/keywords/pixelcnn","display_name":"PixelCNN","score":0.8140000104904175},{"id":"https://openalex.org/keywords/autoencoder","display_name":"autoencoder","score":0.7749999761581421},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.7039999961853027},{"id":"https://openalex.org/keywords/conditional-image-generation","display_name":"conditional image generation","score":0.5460000038146973}],"concepts":[{"id":"https://openalex.org/C60008888","wikidata":"https://www.wikidata.org/wiki/Q6031013","display_name":"Information bottleneck method","level":3,"score":0.9443658590316772},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.7967566251754761},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.7811640501022339},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7805037498474121},{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.6490129232406616},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.6065990328788757},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5840142369270325},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5541853904724121},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5115683674812317},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.48613879084587097},{"id":"https://openalex.org/C152139883","wikidata":"https://www.wikidata.org/wiki/Q252973","display_name":"Mutual information","level":2,"score":0.4696561396121979},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4544851779937744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40147238969802856},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.399812787771225},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.18107232451438904},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2003.06254","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.06254","pdf_url":"https://arxiv.org/pdf/2003.06254","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2003.06254","doi":"https://doi.org/10.48550/arxiv.2003.06254","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2003.06254","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2003.06254","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.06254","pdf_url":"https://arxiv.org/pdf/2003.06254","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4437068315","display_name":"EPSRC Centre for Doctoral Training in Data Science","funder_award_id":"EP/L016427/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2933878393.pdf","grobid_xml":"https://content.openalex.org/works/W2933878393.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W1686946872","https://openalex.org/W2096516049","https://openalex.org/W2108598243","https://openalex.org/W2112796928","https://openalex.org/W2117499988","https://openalex.org/W2194775991","https://openalex.org/W2476548250","https://openalex.org/W2525167219","https://openalex.org/W2529194139","https://openalex.org/W2557579533","https://openalex.org/W2581236139","https://openalex.org/W2593634001","https://openalex.org/W2709553318","https://openalex.org/W2741952635","https://openalex.org/W2777138330","https://openalex.org/W2785885194","https://openalex.org/W2899771611","https://openalex.org/W2950220847","https://openalex.org/W2952502547","https://openalex.org/W2962949867","https://openalex.org/W2963029008","https://openalex.org/W2963636093","https://openalex.org/W2964160479","https://openalex.org/W3118608800"],"related_works":["https://openalex.org/W3210254152","https://openalex.org/W3166094155","https://openalex.org/W2785885194","https://openalex.org/W4410293100","https://openalex.org/W2593634001","https://openalex.org/W2895876826","https://openalex.org/W3174594436","https://openalex.org/W4310497728","https://openalex.org/W2925924375","https://openalex.org/W2906889882"],"abstract_inverted_index":{"The":[0,35],"information":[1,24,48,79],"bottleneck":[2,49],"principle":[3,50],"(Shwartz-Ziv":[4],"&":[5],"Tishby,":[6],"2017)":[7],"suggests":[8],"that":[9,100,111],"SGD-based":[10],"training":[11,108],"of":[12,37,83,103],"deep":[13],"neural":[14],"networks":[15],"results":[16],"in":[17],"optimally":[18],"compressed":[19],"hidden":[20,81,124],"layers,":[21],"from":[22],"an":[23,117,128],"theoretic":[25],"perspective.":[26],"However,":[27],"this":[28],"claim":[29],"was":[30],"established":[31],"on":[32,123],"toy":[33],"data.":[34],"goal":[36],"the":[38,47,77],"work":[39],"we":[40],"present":[41],"here":[42],"is":[43,51],"to":[44,53,75,131,137],"test":[45],"whether":[46],"applicable":[52],"a":[54,58,64,84,134],"realistic":[55],"setting":[56],"using":[57],"larger":[59],"and":[60,86,95,110],"deeper":[61],"convolutional":[62],"architecture,":[63],"ResNet":[65,85],"model.":[66],"We":[67,98],"trained":[68,91],"PixelCNN++":[69],"models":[70],"as":[71],"inverse":[72],"representation":[73],"decoders":[74],"measure":[76],"mutual":[78],"between":[80],"layers":[82],"input":[87],"image":[88],"data,":[89],"when":[90],"for":[92,106,116],"(1)":[93],"classification":[94],"(2)":[96],"autoencoding.":[97],"find":[99],"two":[101],"stages":[102],"learning":[104],"happen":[105],"both":[107],"regimes,":[109],"compression":[112],"does":[113],"occur,":[114],"even":[115],"autoencoder.":[118],"Sampling":[119],"images":[120],"by":[121],"conditioning":[122],"layers'":[125],"activations":[126],"offers":[127],"intuitive":[129],"visualisation":[130],"understand":[132],"what":[133],"ResNets":[135],"learns":[136],"forget.":[138]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2019-04-11T00:00:00"}