{"id":"https://openalex.org/W3126340734","doi":null,"title":"Unpacking Information Bottlenecks: Surrogate Objectives for Deep Learning","display_name":"Unpacking Information Bottlenecks: Surrogate Objectives for Deep Learning","publication_year":2021,"publication_date":"2021-05-04","ids":{"openalex":"https://openalex.org/W3126340734","mag":"3126340734"},"language":"en","primary_location":{"id":"mag:3126340734","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=5rc0K0ezhqI","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Andreas Kirsch","orcid":"https://orcid.org/0000-0003-3578-7504","observed_orcids":["https://orcid.org/0000-0003-3578-7504"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Andreas Kirsch","raw_affiliation_strings":["University of Oxford"],"raw_orcid":"https://orcid.org/0000-0003-3578-7504","affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089486474","display_name":"Clare Lyle","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Clare Lyle","raw_affiliation_strings":["University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029186201","display_name":"Yarin Gal","orcid":"https://orcid.org/0000-0002-2733-2078","observed_orcids":["https://orcid.org/0000-0002-2733-2078"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yarin Gal","raw_affiliation_strings":["University of Oxford"],"raw_orcid":"https://orcid.org/0000-0002-2733-2078","affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I40120149"],"apc_list":null,"apc_paid":null,"fwci":0.0722,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.333939,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":93},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.2581000030040741,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.2581000030040741,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.25760000944137573,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.2029999941587448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/information-bottleneck","display_name":"information bottleneck","score":0.8930000066757202},{"id":"https://openalex.org/keywords/surrogate-objective","display_name":"surrogate objective","score":0.9350000023841858},{"id":"https://openalex.org/keywords/deep-neural-network-training","display_name":"deep neural network training","score":0.7910000085830688},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.8050000071525574},{"id":"https://openalex.org/keywords/regularization","display_name":"regularization","score":0.41100001335144043},{"id":"https://openalex.org/keywords/resnet","display_name":"ResNet","score":0.5770000219345093},{"id":"https://openalex.org/keywords/density-estimation","display_name":"density estimation","score":0.3709999918937683}],"concepts":[{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.8813433647155762},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.8398325443267822},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7217822670936584},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6606553196907043},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6578907370567322},{"id":"https://openalex.org/C60008888","wikidata":"https://www.wikidata.org/wiki/Q6031013","display_name":"Information bottleneck method","level":3,"score":0.5921051502227783},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5816823244094849},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5441386103630066},{"id":"https://openalex.org/C2777256151","wikidata":"https://www.wikidata.org/wiki/Q7897273","display_name":"Unpacking","level":2,"score":0.4727160334587097},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4306575059890747},{"id":"https://openalex.org/C152139883","wikidata":"https://www.wikidata.org/wiki/Q252973","display_name":"Mutual information","level":2,"score":0.22590655088424683},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3126340734","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=5rc0K0ezhqI","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W4919037","https://openalex.org/W582134693","https://openalex.org/W1495576683","https://openalex.org/W1529808766","https://openalex.org/W1596936080","https://openalex.org/W1995875735","https://openalex.org/W2086161653","https://openalex.org/W2112796928","https://openalex.org/W2146705998","https://openalex.org/W2194775991","https://openalex.org/W2240563783","https://openalex.org/W2243397390","https://openalex.org/W2302255633","https://openalex.org/W2552767274","https://openalex.org/W2620998106","https://openalex.org/W2753738274","https://openalex.org/W2785885194","https://openalex.org/W2786858562","https://openalex.org/W2880214242","https://openalex.org/W2902227449","https://openalex.org/W2963334011","https://openalex.org/W2964153729","https://openalex.org/W2970971581","https://openalex.org/W3006100954","https://openalex.org/W3022436500","https://openalex.org/W3118608800","https://openalex.org/W3140968660"],"related_works":["https://openalex.org/W3013360361","https://openalex.org/W3146969938","https://openalex.org/W2789019380","https://openalex.org/W4237649573","https://openalex.org/W4402741181","https://openalex.org/W3160688828","https://openalex.org/W3189482654","https://openalex.org/W6923019291","https://openalex.org/W2982402336","https://openalex.org/W4226315290"],"abstract_inverted_index":{"The":[0],"Information":[1],"Bottleneck":[2],"principle":[3],"offers":[4],"both":[5],"a":[6,20,61],"mechanism":[7],"to":[8,25,47,80,86,105,110],"explain":[9],"how":[10],"deep":[11,51],"neural":[12,52,112],"networks":[13],"train":[14,26],"and":[15,37,72,122],"generalize,":[16],"as":[17,19,60,94],"well":[18],"regularized":[21],"objective":[22],"with":[23,124],"which":[24,54,77],"models.":[27],"However,":[28],"multiple":[29],"competing":[30],"objectives":[31,44,83,102],"are":[32,45],"proposed":[33,75],"in":[34,42,55],"the":[35,38,107],"literature,":[36],"information-theoretic":[39],"quantities":[40],"used":[41],"these":[43,69,100],"difficult":[46],"compute":[48],"for":[49],"large":[50],"networks,":[53],"turn":[56],"limits":[57],"their":[58],"use":[59],"training":[62],"objective.":[63],"In":[64],"this":[65],"work,":[66],"we":[67],"review":[68],"quantities,":[70],"compare":[71],"unify":[73],"previously":[74],"objectives,":[76],"allows":[78],"us":[79,104],"develop":[81],"surrogate":[82,101],"more":[84],"friendly":[85],"optimization":[87],"without":[88],"relying":[89],"on":[90,119],"cumbersome":[91],"tools":[92],"such":[93],"density":[95],"estimation.":[96],"We":[97,115],"find":[98],"that":[99],"allow":[103],"apply":[106],"information":[108],"bottleneck":[109],"modern":[111,125],"network":[113],"architectures.":[114],"demonstrate":[116],"our":[117],"insights":[118],"MNIST,":[120],"CIFAR-10":[121],"ImageNette":[123],"DNN":[126],"architectures":[127],"(ResNets).":[128]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2021-02-15T00:00:00"}