{"id":"https://openalex.org/W4393170742","doi":"https://doi.org/10.1109/tpami.2024.3380007","title":"Designing Universally-Approximating Deep Neural Networks: A First-Order Optimization Approach","display_name":"Designing Universally-Approximating Deep Neural Networks: A First-Order Optimization Approach","publication_year":2024,"publication_date":"2024-03-25","ids":{"openalex":"https://openalex.org/W4393170742","doi":"https://doi.org/10.1109/tpami.2024.3380007","pmid":"https://pubmed.ncbi.nlm.nih.gov/38526901"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2024.3380007","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3380007","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000504407","display_name":"Zhoutong Wu","orcid":"https://orcid.org/0009-0005-6137-5492"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zhoutong Wu","raw_affiliation_strings":["Center for Data Science, Academy for Advanced Interdisciplinary Studies, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0005-6137-5492","affiliations":[{"raw_affiliation_string":"Center for Data Science, Academy for Advanced Interdisciplinary Studies, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075482809","display_name":"Xiao Ming-qing","orcid":"https://orcid.org/0000-0001-6191-7726"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingqing Xiao","raw_affiliation_strings":["National Key Lab of General AI, School of Intelligence Science and Technology, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6191-7726","affiliations":[{"raw_affiliation_string":"National Key Lab of General AI, School of Intelligence Science and Technology, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008843158","display_name":"Cong Fang","orcid":"https://orcid.org/0000-0002-5076-7897"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cong Fang","raw_affiliation_strings":["National Key Lab of General AI, School of Intelligence Science and Technology, Institute for Artificial Intelligence, Peking University, Beijing, China","Institute for Artificial Intelligence, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5076-7897","affiliations":[{"raw_affiliation_string":"National Key Lab of General AI, School of Intelligence Science and Technology, Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I20231570"]},{"raw_affiliation_string":"Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016399094","display_name":"Zhouchen Lin","orcid":"https://orcid.org/0000-0003-1493-7569"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhouchen Lin","raw_affiliation_strings":["National Key Lab of General AI, School of Intelligence Science and Technology, Institute for Artificial Intelligence, Peking University, Beijing, China","Institute for Artificial Intelligence, Peking University, Beijing, China","Peng Cheng Laboratory, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-1493-7569","affiliations":[{"raw_affiliation_string":"National Key Lab of General AI, School of Intelligence Science and Technology, Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I20231570"]},{"raw_affiliation_string":"Institute for Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I20231570"]},{"raw_affiliation_string":"Peng Cheng Laboratory, Guangzhou, China","institution_ids":["https://openalex.org/I4210136793"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5000504407"],"corresponding_institution_ids":["https://openalex.org/I20231570"],"apc_list":null,"apc_paid":null,"fwci":2.3173,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.89130939,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"46","issue":"9","first_page":"6231","last_page":"6246"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/universality","display_name":"Universality (dynamical systems)","score":0.797244668006897},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6815086007118225},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6270595192909241},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5747519135475159},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5647308230400085},{"id":"https://openalex.org/keywords/bounded-function","display_name":"Bounded function","score":0.49973201751708984},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.4368470311164856},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.4154437184333801},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.41353267431259155},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.39316701889038086},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3372798562049866},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.22338849306106567}],"concepts":[{"id":"https://openalex.org/C183992945","wikidata":"https://www.wikidata.org/wiki/Q2495574","display_name":"Universality (dynamical systems)","level":2,"score":0.797244668006897},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6815086007118225},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6270595192909241},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5747519135475159},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5647308230400085},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.49973201751708984},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4368470311164856},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4154437184333801},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.41353267431259155},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.39316701889038086},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3372798562049866},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22338849306106567},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2024.3380007","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3380007","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:38526901","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38526901","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G870971120","display_name":null,"funder_award_id":"2022ZD0160300","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":94,"referenced_works":["https://openalex.org/W605934366","https://openalex.org/W1536680647","https://openalex.org/W1677182931","https://openalex.org/W1988720110","https://openalex.org/W2060270693","https://openalex.org/W2092272609","https://openalex.org/W2097117768","https://openalex.org/W2097460613","https://openalex.org/W2103496339","https://openalex.org/W2108598243","https://openalex.org/W2137983211","https://openalex.org/W2194775991","https://openalex.org/W2302255633","https://openalex.org/W2317926736","https://openalex.org/W2525778437","https://openalex.org/W2529992474","https://openalex.org/W2549139847","https://openalex.org/W2600297185","https://openalex.org/W2738743584","https://openalex.org/W2743909828","https://openalex.org/W2746314669","https://openalex.org/W2749772809","https://openalex.org/W2763081248","https://openalex.org/W2766099245","https://openalex.org/W2766447205","https://openalex.org/W2902719825","https://openalex.org/W2963446712","https://openalex.org/W2964102336","https://openalex.org/W2964258058","https://openalex.org/W2974916071","https://openalex.org/W2982376398","https://openalex.org/W2995681639","https://openalex.org/W3002335888","https://openalex.org/W3034429256","https://openalex.org/W3099132019","https://openalex.org/W3102511045","https://openalex.org/W3103586216","https://openalex.org/W3110996603","https://openalex.org/W3118608800","https://openalex.org/W3125537303","https://openalex.org/W3133902371","https://openalex.org/W3138516171","https://openalex.org/W3201167453","https://openalex.org/W4206349042","https://openalex.org/W4231202173","https://openalex.org/W4240391021","https://openalex.org/W4244393449","https://openalex.org/W4255754126","https://openalex.org/W4282919217","https://openalex.org/W4287629778","https://openalex.org/W4297775537","https://openalex.org/W4297801963","https://openalex.org/W4312443924","https://openalex.org/W4378533149","https://openalex.org/W6631943919","https://openalex.org/W6638670064","https://openalex.org/W6675164516","https://openalex.org/W6677645113","https://openalex.org/W6684191040","https://openalex.org/W6691187937","https://openalex.org/W6695533872","https://openalex.org/W6714181750","https://openalex.org/W6727690538","https://openalex.org/W6729972426","https://openalex.org/W6737664043","https://openalex.org/W6739901393","https://openalex.org/W6743428213","https://openalex.org/W6743923790","https://openalex.org/W6745535076","https://openalex.org/W6745764291","https://openalex.org/W6752515464","https://openalex.org/W6752578362","https://openalex.org/W6754662975","https://openalex.org/W6756221748","https://openalex.org/W6757465018","https://openalex.org/W6760905529","https://openalex.org/W6760979436","https://openalex.org/W6762718338","https://openalex.org/W6763000687","https://openalex.org/W6766816269","https://openalex.org/W6767563556","https://openalex.org/W6774132597","https://openalex.org/W6779382319","https://openalex.org/W6779567887","https://openalex.org/W6779581272","https://openalex.org/W6779806301","https://openalex.org/W6780226713","https://openalex.org/W6784333009","https://openalex.org/W6784796509","https://openalex.org/W6787149102","https://openalex.org/W6787972765","https://openalex.org/W6788135285","https://openalex.org/W6790057526","https://openalex.org/W6791002663"],"related_works":["https://openalex.org/W1587914261","https://openalex.org/W2065095781","https://openalex.org/W4295724953","https://openalex.org/W4377865163","https://openalex.org/W3193857078","https://openalex.org/W2888956734","https://openalex.org/W3000197790","https://openalex.org/W4315865067","https://openalex.org/W2979433843","https://openalex.org/W3208304128"],"abstract_inverted_index":{"Universal":[0],"approximation":[1,67,187],"capability,":[2],"also":[3],"referred":[4],"to":[5,21,85,135,250],"as":[6,55,108,210,214,216,241,243],"universality,":[7],"is":[8,106,120,178],"an":[9,112],"important":[10],"property":[11],"of":[12,35,43,89,111,147,161,174,202,238,253],"deep":[13,36,90],"neural":[14,37,52,61,91,149,203],"networks,":[15,150],"endowing":[16],"them":[17],"with":[18,93,130,185],"the":[19,24,33,41,44,56,65,75,87,103,109,128,145,169,172,179,228,236,244,251],"potency":[20],"accurately":[22],"represent":[23],"underlying":[25],"target":[26],"function":[27],"in":[28,127,192],"learning":[29],"tasks.":[30],"In":[31,77],"practice,":[32],"architecture":[34,62],"networks":[38,92,184,231],"largely":[39],"influences":[40],"performance":[42],"models.":[45],"However,":[46],"most":[47],"existing":[48],"methodologies":[49],"for":[50],"designing":[51],"architectures,":[53],"such":[54,209],"heuristic":[57],"manual":[58],"design":[59,86],"or":[60,117,139],"search,":[63],"ignore":[64],"universal":[66,186],"property,":[68],"thus":[69],"losing":[70],"a":[71,82,94,131,136,151,193,200],"potential":[72],"safeguard":[73],"about":[74],"performance.":[76],"this":[78,177],"paper,":[79],"we":[80,143],"propose":[81],"unified":[83],"framework":[84,197],"architectures":[88,204],"universality":[95],"guarantee":[96,188],"based":[97],"on":[98,222],"first-order":[99],"optimization":[100,113],"algorithms,":[101],"where":[102],"forward":[104],"pass":[105],"interpreted":[107],"updates":[110],"algorithm.":[114],"The":[115,219],"(explicit":[116],"implicit)":[118],"network":[119,138],"designed":[121,191],"by":[122],"replacing":[123],"each":[124],"gradient":[125],"term":[126],"algorithm":[129],"learnable":[132],"module":[133],"similar":[134],"two-layer":[137],"its":[140],"derivatives.":[141],"Specifically,":[142],"explore":[144],"realm":[146],"width-bounded":[148,183],"common":[152],"practical":[153],"scenario,":[154],"showcasing":[155],"their":[156],"universality.":[157,170],"Moreover,":[158],"adding":[159],"operations":[160],"normalization,":[162],"downsampling,":[163],"and":[164,212,234,247],"upsampling":[165],"does":[166],"not":[167],"hurt":[168],"To":[171],"best":[173],"our":[175,254],"knowledge,":[176],"first":[180],"work":[181],"that":[182,227],"can":[189,198],"be":[190],"principled":[194],"way.":[195],"Our":[196],"inspire":[199],"variety":[201],"including":[205],"some":[206],"renowned":[207],"structures":[208],"ResNet":[211],"DenseNet,":[213,240],"well":[215,242],"novel":[217],"innovations.":[218],"experimental":[220],"results":[221],"image":[223],"classification":[224],"problems":[225],"demonstrate":[226],"newly":[229],"inspired":[230],"are":[232],"competitive":[233],"surpass":[235],"baselines":[237],"ResNet,":[239],"advanced":[245],"ConvNeXt":[246],"ViT,":[248],"testifying":[249],"effectiveness":[252],"framework.":[255]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":3}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-10-10T00:00:00"}