{"id":"https://openalex.org/W2906889882","doi":"https://doi.org/10.48550/arxiv.1902.09037","title":"Adaptive Estimators Show Information Compression in Deep Neural Networks","display_name":"Adaptive Estimators Show Information Compression in Deep Neural Networks","alternate_titles":[],"publication_year":2019,"publication_date":"2019-02-24","ids":{"openalex":"https://openalex.org/W2906889882","doi":"https://doi.org/10.48550/arxiv.1902.09037","mag":"2906889882"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1902.09037","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.09037","pdf_url":"https://arxiv.org/pdf/1902.09037","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W2906889882"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1902.09037","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045216554","display_name":"Ivan Chelombiev","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Chelombiev, Ivan","raw_affiliation_strings":["Department of Computer Science University of Bristol , Bristol , UK","University of Bristol, Bristol, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science University of Bristol , Bristol , UK","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002438004","display_name":"Conor J Houghton","orcid":"https://orcid.org/0000-0001-5017-9473","observed_orcids":["https://orcid.org/0000-0001-5017-9473"]},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Houghton, Conor","raw_affiliation_strings":["Department of Computer Science University of Bristol , Bristol , UK","University of Bristol, Bristol, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science University of Bristol , Bristol , UK","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046112755","display_name":"Cian O\u2019Donnell","orcid":"https://orcid.org/0000-0003-2031-9177","observed_orcids":["https://orcid.org/0000-0003-2031-9177"]},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"O'Donnell, Cian","raw_affiliation_strings":["Department of Computer Science University of Bristol , Bristol , UK","University of Bristol, Bristol, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science University of Bristol , Bristol , UK","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I36234482"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":50,"reviewed_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9458000063896179,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9458000063896179,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.025200000032782555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.007000000216066837,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/information-bottleneck-principle","display_name":"information bottleneck theory","score":0.8479999899864197},{"id":"https://openalex.org/keywords/representation-compression","display_name":"representation compression","score":0.8050000071525574},{"id":"https://openalex.org/keywords/mutual-information-estimation","display_name":"mutual information estimation","score":0.9570000171661377},{"id":"https://openalex.org/keywords/adaptive-estimators","display_name":"adaptive estimators","score":0.7990000247955322},{"id":"https://openalex.org/keywords/activation-functions","display_name":"activation functions","score":0.6110000014305115},{"id":"https://openalex.org/keywords/neural-network-generalization","display_name":"neural network generalization","score":0.578000009059906},{"id":"https://openalex.org/keywords/tikhonov-regularization","display_name":"Tikhonov regularization","score":0.8529999852180481},{"id":"https://openalex.org/keywords/network-initialization","display_name":"network initialization","score":0.9350000023841858},{"id":"https://openalex.org/keywords/deep-neural-network","display_name":"deep neural network","score":0.5860000252723694}],"concepts":[{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.766545295715332},{"id":"https://openalex.org/C60008888","wikidata":"https://www.wikidata.org/wiki/Q6031013","display_name":"Information bottleneck method","level":3,"score":0.7405444979667664},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7018595337867737},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6545861959457397},{"id":"https://openalex.org/C38365724","wikidata":"https://www.wikidata.org/wiki/Q4677469","display_name":"Activation function","level":3,"score":0.6265946626663208},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.571378231048584},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.5506014823913574},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5404667854309082},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.5398226976394653},{"id":"https://openalex.org/C152139883","wikidata":"https://www.wikidata.org/wiki/Q252973","display_name":"Mutual information","level":2,"score":0.3971337378025055},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.32646453380584717},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21777012944221497},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:1902.09037","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.09037","pdf_url":"https://arxiv.org/pdf/1902.09037","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2950850913","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1902.09037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1902.09037","doi":"https://doi.org/10.48550/arxiv.1902.09037","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1902.09037","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:2906889882","doi":null,"is_oa":true,"landing_page_url":"https://dblp.uni-trier.de/db/conf/iclr/iclr2019.html#ChelombievHO19","pdf_url":null,"source":{"id":"https://openalex.org/S4306419637","display_name":"International Conference on Learning Representations","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Learning Representations","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1902.09037","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.09037","pdf_url":"https://arxiv.org/pdf/1902.09037","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G6645945671","display_name":"How is sensory information encoded in spike trains?","funder_award_id":"220020239","funder_id":"https://openalex.org/F4320306183","funder_display_name":"James S. McDonnell Foundation"}],"funders":[{"id":"https://openalex.org/F4320306183","display_name":"James S. McDonnell Foundation","ror":"https://ror.org/03dy4aq19"},{"id":"https://openalex.org/F4320320360","display_name":"University of Bristol","ror":"https://ror.org/0524sp257"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2906889882.pdf","grobid_xml":"https://content.openalex.org/works/W2906889882.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W69783631","https://openalex.org/W1522301498","https://openalex.org/W1533861849","https://openalex.org/W1665214252","https://openalex.org/W1677182931","https://openalex.org/W1686946872","https://openalex.org/W2076063813","https://openalex.org/W2096516049","https://openalex.org/W2098145033","https://openalex.org/W2156387975","https://openalex.org/W2176412452","https://openalex.org/W2418033038","https://openalex.org/W2593634001","https://openalex.org/W2622719862","https://openalex.org/W2683470288","https://openalex.org/W2766447205","https://openalex.org/W2767286248","https://openalex.org/W2785648721","https://openalex.org/W2785885194","https://openalex.org/W2950220847","https://openalex.org/W2962834855"],"related_works":["https://openalex.org/W2785885194","https://openalex.org/W4384301102","https://openalex.org/W7203568733","https://openalex.org/W3174594436","https://openalex.org/W2895876826","https://openalex.org/W2995631573","https://openalex.org/W2788440360","https://openalex.org/W7212381432","https://openalex.org/W4410293100","https://openalex.org/W4376653428"],"abstract_inverted_index":{"To":[0],"improve":[1],"how":[2],"neural":[3,23,96],"networks":[4,24,56,63,97,120],"function":[5,142],"it":[6],"is":[7,36,48,143,163,199],"crucial":[8],"to":[9,32,39,92,182],"understand":[10],"their":[11,30],"learning":[12,20],"process.":[13],"The":[14],"information":[15,34,87],"bottleneck":[16],"theory":[17,47],"of":[18,71,95,103,124,132,139,151,167,195],"deep":[19],"proposes":[21],"that":[22,35,90,137,161,178,192],"achieve":[25],"good":[26],"generalization":[27],"by":[28],"compressing":[29],"representations":[31],"disregard":[33],"not":[37,76,144],"relevant":[38],"the":[40,140,149,196],"task.":[41],"However,":[42],"empirical":[43],"evidence":[44],"for":[45,146],"this":[46,80],"conflicting,":[49],"as":[50],"compression":[51,118,152,170,194],"was":[52],"only":[53,193],"observed":[54],"when":[55],"used":[57],"saturating":[58],"activation":[59,66,126,141,156],"functions.":[60,110,127,157],"In":[61,79],"contrast,":[62],"with":[64,121,202],"non-saturating":[65],"functions":[67],"achieved":[68],"comparable":[69],"levels":[70],"task":[72],"performance":[73],"but":[74],"did":[75],"show":[77,136,191],"compression.":[78],"paper":[81],"we":[82,116,135,176,190],"developed":[83],"more":[84,100],"robust":[85],"mutual":[86],"estimation":[88,114],"techniques,":[89,115],"adapt":[91],"hidden":[93],"activity":[94],"and":[98,148],"produce":[99],"sensitive":[101],"measurements":[102],"activations":[104],"from":[105],"all":[106],"functions,":[107],"especially":[108],"unbounded":[109],"Using":[111],"these":[112],"adaptive":[113],"explored":[117],"in":[119,169],"a":[122,164],"range":[123],"different":[125,155,172],"With":[128],"two":[129],"improved":[130],"methods":[131],"estimation,":[133],"firstly,":[134],"saturation":[138],"required":[145],"compression,":[147,185],"amount":[150,166],"varies":[153],"between":[154,171],"We":[158],"also":[159],"find":[160],"there":[162],"large":[165],"variation":[168],"network":[173],"initializations.":[174],"Secondary,":[175],"see":[177],"L2":[179],"regularization":[180],"leads":[181],"significantly":[183],"increased":[184],"while":[186],"preventing":[187],"overfitting.":[188],"Finally,":[189],"last":[197],"layer":[198],"positively":[200],"correlated":[201],"generalization.":[203]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2906889882","counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":2}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2019-01-11T00:00:00"}
