{"id":"https://openalex.org/W3199247446","doi":"https://doi.org/10.1109/tnnls.2021.3109942","title":"An Optimal Transport Analysis on Generalization in Deep Learning","display_name":"An Optimal Transport Analysis on Generalization in Deep Learning","publication_year":2021,"publication_date":"2021-09-23","ids":{"openalex":"https://openalex.org/W3199247446","doi":"https://doi.org/10.1109/tnnls.2021.3109942","mag":"3199247446","pmid":"https://pubmed.ncbi.nlm.nih.gov/34554918"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2021.3109942","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3109942","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100434263","display_name":"Jingwei Zhang","orcid":"https://orcid.org/0000-0002-1681-6608"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jingwei Zhang","raw_affiliation_strings":["Department of Computer Science and Engineering, School of Engineering, The Hong Kong University of Science and Technology, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-1681-6608","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, School of Engineering, The Hong Kong University of Science and Technology, Hong Kong","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065250332","display_name":"Tongliang Liu","orcid":"https://orcid.org/0000-0002-9640-6472"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Tongliang Liu","raw_affiliation_strings":["Faculty of Engineering, School of Computer Science, The University of Sydney, Darlington, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0002-9640-6472","affiliations":[{"raw_affiliation_string":"Faculty of Engineering, School of Computer Science, The University of Sydney, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101409300","display_name":"Dacheng Tao","orcid":"https://orcid.org/0000-0002-5979-578X"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Dacheng Tao","raw_affiliation_strings":["School of Computer Science, Faculty of Engineering, The University of Sydney, Darlington, NSW, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Faculty of Engineering, The University of Sydney, Darlington, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.5013,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.8556547,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"34","issue":"6","first_page":"2842","last_page":"2853"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12303","display_name":"Tensor decomposition and applications","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/2605","display_name":"Computational Mathematics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.87535560131073},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6992114782333374},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6180485486984253},{"id":"https://openalex.org/keywords/generalization-error","display_name":"Generalization error","score":0.6088082790374756},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5855916142463684},{"id":"https://openalex.org/keywords/early-stopping","display_name":"Early stopping","score":0.4688493013381958},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.4574874937534332},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4563820958137512},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44940513372421265},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4485425651073456},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2207244336605072}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.87535560131073},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6992114782333374},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6180485486984253},{"id":"https://openalex.org/C117765406","wikidata":"https://www.wikidata.org/wiki/Q5362437","display_name":"Generalization error","level":3,"score":0.6088082790374756},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5855916142463684},{"id":"https://openalex.org/C5465570","wikidata":"https://www.wikidata.org/wiki/Q5326898","display_name":"Early stopping","level":3,"score":0.4688493013381958},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.4574874937534332},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4563820958137512},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44940513372421265},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4485425651073456},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2207244336605072},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tnnls.2021.3109942","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3109942","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:34554918","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/34554918","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-114022","is_oa":false,"landing_page_url":"https://repository.hkust.edu.hk/ir/Record/1783.1-114022","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},{"id":"pmh:oai:repository.ust.hk:1783.1-114022","is_oa":false,"landing_page_url":"http://gateway.isiknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcAuth=LinksAMR&SrcApp=PARTNER_APP&DestLinkType=FullRecord&DestApp=WOS&KeyUT=000636274000038","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5299999713897705,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G5711944514","display_name":null,"funder_award_id":"DP-180103424","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"},{"id":"https://openalex.org/G7519899501","display_name":null,"funder_award_id":"FL-170100117","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"},{"id":"https://openalex.org/G7983224991","display_name":null,"funder_award_id":"IH-180100002","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":68,"referenced_works":["https://openalex.org/W27434444","https://openalex.org/W385466589","https://openalex.org/W1499021337","https://openalex.org/W1594039573","https://openalex.org/W1663583528","https://openalex.org/W1892947258","https://openalex.org/W1968436459","https://openalex.org/W2063368770","https://openalex.org/W2112065930","https://openalex.org/W2149298154","https://openalex.org/W2167453047","https://openalex.org/W2188409822","https://openalex.org/W2210387432","https://openalex.org/W2258658829","https://openalex.org/W2290247654","https://openalex.org/W2294659999","https://openalex.org/W2378248953","https://openalex.org/W2518143623","https://openalex.org/W2540093921","https://openalex.org/W2541283300","https://openalex.org/W2579923771","https://openalex.org/W2592206372","https://openalex.org/W2617539226","https://openalex.org/W2623154256","https://openalex.org/W2739748921","https://openalex.org/W2774276467","https://openalex.org/W2793477525","https://openalex.org/W2808225711","https://openalex.org/W2886919371","https://openalex.org/W2912849082","https://openalex.org/W2914852400","https://openalex.org/W2951714068","https://openalex.org/W2963103083","https://openalex.org/W2972997443","https://openalex.org/W2997053319","https://openalex.org/W3009557165","https://openalex.org/W3102159535","https://openalex.org/W3113070793","https://openalex.org/W3137695714","https://openalex.org/W3153756118","https://openalex.org/W4251333191","https://openalex.org/W4293583552","https://openalex.org/W4295521014","https://openalex.org/W4298090544","https://openalex.org/W4306353376","https://openalex.org/W6636906085","https://openalex.org/W6676765452","https://openalex.org/W6684836731","https://openalex.org/W6687056409","https://openalex.org/W6692563690","https://openalex.org/W6697233702","https://openalex.org/W6703180097","https://openalex.org/W6732517885","https://openalex.org/W6734105973","https://openalex.org/W6735913928","https://openalex.org/W6738074204","https://openalex.org/W6739002271","https://openalex.org/W6741832134","https://openalex.org/W6748601875","https://openalex.org/W6750582233","https://openalex.org/W6752717554","https://openalex.org/W6754046860","https://openalex.org/W6754055986","https://openalex.org/W6759238893","https://openalex.org/W6763485134","https://openalex.org/W6787039732","https://openalex.org/W7034108470","https://openalex.org/W7046297837"],"related_works":["https://openalex.org/W2381411913","https://openalex.org/W2010541108","https://openalex.org/W2087331387","https://openalex.org/W2006060538","https://openalex.org/W4367000426","https://openalex.org/W4321151660","https://openalex.org/W4386047771","https://openalex.org/W2107708607","https://openalex.org/W4283332751","https://openalex.org/W4367060723"],"abstract_inverted_index":{"Deep":[0],"neural":[1],"networks":[2],"(DNNs)":[3],"have":[4],"achieved":[5],"state-of-the-art":[6],"performance":[7],"in":[8,31,45,115,128,170,182,202,255,319],"various":[9],"learning":[10,63,90,129,141,209,220,283],"tasks,":[11],"such":[12,102],"as":[13,103,325],"computer":[14],"vision,":[15],"natural":[16],"language":[17],"processing,":[18],"and":[19,48,143,147,159,235,263],"speech":[20],"recognition.":[21],"However,":[22],"the":[23,59,68,82,140,144,149,163,216,227,232,236,251,278,297,301,316,326],"fundamental":[24,178],"theory":[25,56,154],"of":[26,57,62,70,77,86,113,133,166,196,219,257,281,328],"generalization":[27,60,69,87,114,134,164,217,279,317],"still":[28],"remains":[29],"obscure":[30],"deep":[32,183],"learning-why":[33,184],"DNN":[34],"models":[35,64,91,97,186],"can":[36,155,187,199,285],"generalize":[37,188],"well,":[38],"despite":[39],"that":[40,54,81,315],"they":[41],"are":[42],"heavily":[43],"overparametrized":[44],"both":[46,139],"depth":[47],"width?":[49],"Recently,":[50],"some":[51],"work":[52],"shows":[53],"traditional":[55,123],"analyzing":[58],"error":[61,88,135,218,280,318],"fails":[65],"to":[66,161,175,296,323],"explain":[67],"DNNs.":[71,104,167],"The":[72,193],"failure":[73],"is":[74,136,148],"mainly":[75],"because":[76],"one":[78],"simple":[79],"fact":[80],"worse":[83],"case":[84],"analysis":[85,112,127,132],"for":[89,96,273],"would":[92],"be":[93,156,200,286],"too":[94],"loose":[95],"with":[98],"large":[99],"parameter":[100],"space,":[101],"In":[105],"this":[106,171,197],"work,":[107],"we":[108,173,211,268,309],"propose":[109],"a":[110,177,207,282],"new":[111],"DNNs":[116,320],"from":[117],"an":[118,241],"optimal":[119],"transport":[120,225,253],"perspective.":[121],"Unlike":[122],"worse-case":[124],"uniform":[125],"convergence":[126],"theory,":[130],"our":[131,153,306,311],"dependent":[137],"on":[138,215,240,250],"algorithm":[142,284],"data":[145],"distribution":[146],"average-case":[150],"analysis.":[151],"Thus,":[152],"more":[157],"practical":[158],"accurate":[160],"describe":[162],"behavior":[165],"More":[168],"specifically,":[169],"article,":[172],"try":[174],"answer":[176],"yet":[179],"unsolved":[180],"question":[181],"deeper":[185],"well":[189],"than":[190],"shallow":[191],"models?":[192],"main":[194,312],"contribution":[195],"article":[198],"summarized":[201],"four":[203],"aspects.":[204],"First,":[205],"under":[206,276,305],"general":[208],"framework,":[210,308],"derive":[212],"upper":[213,248,287],"bounds":[214,249],"algorithms":[221],"by":[222,289],"their":[223],"algorithmic":[224,252],"cost:":[226],"expected":[228],"Wasserstein":[229],"distance":[230],"between":[231,293],"output":[233,237,298],"hypothesis":[234,238,299],"conditioned":[239],"input":[242,302],"example.":[243],"We":[244],"further":[245],"provide":[246],"several":[247],"cost":[254],"terms":[256],"total":[258],"variation":[259],"distance,":[260],"relative":[261],"entropy,":[262],"Vapnik-Chervonenkis":[264],"(VC)":[265],"dimension.":[266],"Moreover,":[267],"also":[269],"study":[270],"different":[271,290],"conditions":[272],"loss":[274],"functions":[275],"which":[277],"bounded":[288],"probability":[291],"metrics":[292],"distributions":[294],"relating":[295],"and/or":[300],"data.":[303],"Finally,":[304],"established":[307],"obtain":[310],"results,":[313],"showing":[314],"decreases":[321],"exponentially":[322],"zero":[324],"number":[327],"layers":[329],"increases.":[330]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}