{"id":"https://openalex.org/W4385864301","doi":"https://doi.org/10.1109/tnnls.2023.3297113","title":"Going Deeper, Generalizing Better: An Information-Theoretic View for Deep Learning","display_name":"Going Deeper, Generalizing Better: An Information-Theoretic View for Deep Learning","publication_year":2023,"publication_date":"2023-08-16","ids":{"openalex":"https://openalex.org/W4385864301","doi":"https://doi.org/10.1109/tnnls.2023.3297113","pmid":"https://pubmed.ncbi.nlm.nih.gov/37585328"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2023.3297113","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3297113","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100434263","display_name":"Jingwei Zhang","orcid":"https://orcid.org/0000-0002-1681-6608"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jingwei Zhang","raw_affiliation_strings":["Department of Computer Science and Engineering, School of Engineering, The Hong Kong University of Science and Technology, Sai Kung, Hong Kong","Department of Computer Science and Engineering, School of Engineering, The Hong Kong University of Science and Technology, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-1681-6608","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, School of Engineering, The Hong Kong University of Science and Technology, Sai Kung, Hong Kong","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, School of Engineering, The Hong Kong University of Science and Technology, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065250332","display_name":"Tongliang Liu","orcid":"https://orcid.org/0000-0002-9640-6472"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Tongliang Liu","raw_affiliation_strings":["Sydney AI Centre, the chool of Computer Science, and the Faculty of Engineering, The Univerity of Sydney, Darlington, NSW, Australia","Sydney AI Centre, the School of Computer Science, and the Faculty of Engineering, The University of Sydney, Darlington, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0002-9640-6472","affiliations":[{"raw_affiliation_string":"Sydney AI Centre, the chool of Computer Science, and the Faculty of Engineering, The Univerity of Sydney, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]},{"raw_affiliation_string":"Sydney AI Centre, the School of Computer Science, and the Faculty of Engineering, The University of Sydney, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074103823","display_name":"Dacheng Tao","orcid":"https://orcid.org/0000-0001-7225-5449"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Dacheng Tao","raw_affiliation_strings":["Sydney AI Centre, the chool of Computer Science, and the Faculty of Engineering, The Univerity of Sydney, Darlington, NSW, Australia","Sydney AI Centre, the School of Computer Science, and the Faculty of Engineering, The University of Sydney, Darlington, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7225-5449","affiliations":[{"raw_affiliation_string":"Sydney AI Centre, the chool of Computer Science, and the Faculty of Engineering, The Univerity of Sydney, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]},{"raw_affiliation_string":"Sydney AI Centre, the School of Computer Science, and the Faculty of Engineering, The University of Sydney, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7449,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.87019275,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"35","issue":"11","first_page":"16683","last_page":"16695"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.7921004295349121},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.7522844076156616},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6872414946556091},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6186407208442688},{"id":"https://openalex.org/keywords/generalization-error","display_name":"Generalization error","score":0.5603177547454834},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5576093196868896},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5407986044883728},{"id":"https://openalex.org/keywords/early-stopping","display_name":"Early stopping","score":0.47613075375556946},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.4742250144481659},{"id":"https://openalex.org/keywords/bounded-function","display_name":"Bounded function","score":0.4727112948894501},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.47132351994514465},{"id":"https://openalex.org/keywords/word-error-rate","display_name":"Word error rate","score":0.46311676502227783},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.42812204360961914},{"id":"https://openalex.org/keywords/upper-and-lower-bounds","display_name":"Upper and lower bounds","score":0.41170692443847656},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.39619821310043335},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3460734486579895},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2574306130409241},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.1481403112411499}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7921004295349121},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.7522844076156616},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6872414946556091},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6186407208442688},{"id":"https://openalex.org/C117765406","wikidata":"https://www.wikidata.org/wiki/Q5362437","display_name":"Generalization error","level":3,"score":0.5603177547454834},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5576093196868896},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5407986044883728},{"id":"https://openalex.org/C5465570","wikidata":"https://www.wikidata.org/wiki/Q5326898","display_name":"Early stopping","level":3,"score":0.47613075375556946},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.4742250144481659},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.4727112948894501},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.47132351994514465},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.46311676502227783},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.42812204360961914},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.41170692443847656},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.39619821310043335},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3460734486579895},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2574306130409241},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.1481403112411499},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tnnls.2023.3297113","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2023.3297113","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:37585328","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37585328","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-129762","is_oa":false,"landing_page_url":"https://repository.hkust.edu.hk/ir/Record/1783.1-129762","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6299999952316284}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W27434444","https://openalex.org/W1544131626","https://openalex.org/W1567512734","https://openalex.org/W2478708596","https://openalex.org/W2527691228","https://openalex.org/W2540093921","https://openalex.org/W2541283300","https://openalex.org/W2612690371","https://openalex.org/W2766678531","https://openalex.org/W2900959181","https://openalex.org/W2962702650","https://openalex.org/W2963095610","https://openalex.org/W2963239103","https://openalex.org/W2963791871","https://openalex.org/W3101806332","https://openalex.org/W4232613155","https://openalex.org/W4289293816","https://openalex.org/W4289436733","https://openalex.org/W4293774175","https://openalex.org/W4297749952","https://openalex.org/W6601076486","https://openalex.org/W6679702791","https://openalex.org/W6682631133","https://openalex.org/W6687056409","https://openalex.org/W6728598543","https://openalex.org/W6733862737","https://openalex.org/W6738074204","https://openalex.org/W6738336337","https://openalex.org/W6745256532","https://openalex.org/W6751754507","https://openalex.org/W6752009368","https://openalex.org/W6752495264","https://openalex.org/W6753918066","https://openalex.org/W6754984521","https://openalex.org/W6755150206","https://openalex.org/W6756001544","https://openalex.org/W6756091659","https://openalex.org/W6756137178","https://openalex.org/W6756455746","https://openalex.org/W6758902322","https://openalex.org/W6759238893","https://openalex.org/W6785968470","https://openalex.org/W6796672888"],"related_works":["https://openalex.org/W2136627871","https://openalex.org/W2153743018","https://openalex.org/W2117037448","https://openalex.org/W4387929605","https://openalex.org/W2048948323","https://openalex.org/W2109850386","https://openalex.org/W3025192034","https://openalex.org/W3037653749","https://openalex.org/W1588766645","https://openalex.org/W2141212940"],"abstract_inverted_index":{"Deep":[0],"learning":[1,190],"has":[2],"transformed":[3],"computer":[4],"vision,":[5],"natural":[6],"language":[7],"processing,":[8],"and":[9,30,76,197,209],"speech":[10],"recognition.":[11],"However,":[12,131],"two":[13],"critical":[14],"questions":[15],"remain":[16],"obscure:":[17],"1)":[18],"why":[19],"do":[20,138],"deep":[21,189],"neural":[22,56],"networks":[23,29,57],"(DNNs)":[24],"generalize":[25],"better":[26,42],"than":[27],"shallow":[28],"2)":[31],"does":[32],"it":[33],"always":[34],"hold":[35],"that":[36,50,87,171,188],"a":[37,141,181,192],"deeper":[38,175],"network":[39],"leads":[40],"to":[41],"performance?":[43],"In":[44],"this":[45,126],"article,":[46],"we":[47,186],"first":[48,129],"show":[49,187],"the":[51,64,68,72,77,80,89,95,97,113,119,123,128,148,155,159,164,172,176],"expected":[52,98,135,149],"generalization":[53,99,120,136,200],"error":[54,100,121,137,151,201],"of":[55,79,91,166,195],"(NNs)":[58],"can":[59],"be":[60],"upper":[61],"bounded":[62],"by":[63],"mutual":[65],"information":[66,109,156],"between":[67],"learned":[69],"features":[70],"in":[71,94,212],"last":[73],"hidden":[74],"layer":[75],"parameters":[78],"output":[81],"layer.":[82],"This":[83,145,169],"bound":[84],"further":[85],"implies":[86],"as":[88,112,163],"number":[90,165],"layers":[92,167],"increases":[93],"network,":[96],"will":[101],"decrease":[102],"under":[103],"mild":[104],"conditions.":[105],"Layers":[106],"with":[107,133],"strict":[108],"loss,":[110],"such":[111],"convolutional":[114],"or":[115],"pooling":[116],"layers,":[117],"reduce":[118],"for":[122,157,203],"whole":[124],"network;":[125],"answers":[127],"question.":[130],"algorithms":[132],"zero":[134],"not":[139],"imply":[140],"small":[142,182],"test":[143],"error.":[144,184],"is":[146,152,161,178],"because":[147],"training":[150,183],"large":[153],"when":[154],"fitting":[158],"data":[160],"lost":[162],"increases.":[168],"suggests":[170],"claim":[173],"\"the":[174],"better\"":[177],"conditioned":[179],"on":[180],"Finally,":[185],"satisfies":[191],"weak":[193],"notion":[194],"stability":[196],"provides":[198],"some":[199],"bounds":[202],"noisy":[204],"stochastic":[205],"gradient":[206],"decent":[207],"(SGD)":[208],"binary":[210],"classification":[211],"DNNs.":[213]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":5},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}