{"id":"https://openalex.org/W3137190706","doi":"https://doi.org/10.1109/bigdata50022.2020.9378359","title":"Neural Network Training Techniques Regularize Optimization Trajectory: An Empirical Study","display_name":"Neural Network Training Techniques Regularize Optimization Trajectory: An Empirical Study","publication_year":2020,"publication_date":"2020-12-10","ids":{"openalex":"https://openalex.org/W3137190706","doi":"https://doi.org/10.1109/bigdata50022.2020.9378359","mag":"3137190706"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata50022.2020.9378359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata50022.2020.9378359","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2020 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100420548","display_name":"Cheng Chen","orcid":"https://orcid.org/0000-0002-4203-2145"},"institutions":[{"id":"https://openalex.org/I223532165","display_name":"University of Utah","ror":"https://ror.org/03r0ha626","country_code":"US","type":"education","lineage":["https://openalex.org/I223532165"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cheng Chen","raw_affiliation_strings":["University of Utah,Department of ECE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Utah,Department of ECE","institution_ids":["https://openalex.org/I223532165"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089183792","display_name":"Junjie Yang","orcid":"https://orcid.org/0000-0002-1240-6417"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junjie Yang","raw_affiliation_strings":["The Ohio State University,Department of ECE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University,Department of ECE","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101704070","display_name":"Yi Zhou","orcid":"https://orcid.org/0000-0002-3982-9145"},"institutions":[{"id":"https://openalex.org/I223532165","display_name":"University of Utah","ror":"https://ror.org/03r0ha626","country_code":"US","type":"education","lineage":["https://openalex.org/I223532165"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yi Zhou","raw_affiliation_strings":["University of Utah,Department of ECE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Utah,Department of ECE","institution_ids":["https://openalex.org/I223532165"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"70","issue":null,"first_page":"141","last_page":"146"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/regularization","display_name":"Regularization (linguistics)","score":0.8388385772705078},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6973555684089661},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6836602091789246},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.6074307560920715},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.5864886045455933},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.5389472246170044},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5269438028335571},{"id":"https://openalex.org/keywords/nonlinear-system","display_name":"Nonlinear system","score":0.5048490166664124},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.4777604639530182},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45699068903923035},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.4210110306739807},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4040682911872864},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.339491605758667},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.30933046340942383},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.22842466831207275}],"concepts":[{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.8388385772705078},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6973555684089661},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6836602091789246},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.6074307560920715},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.5864886045455933},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.5389472246170044},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5269438028335571},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.5048490166664124},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.4777604639530182},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45699068903923035},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.4210110306739807},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4040682911872864},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.339491605758667},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.30933046340942383},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22842466831207275},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata50022.2020.9378359","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata50022.2020.9378359","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2020 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W1498436455","https://openalex.org/W1522301498","https://openalex.org/W1665214252","https://openalex.org/W1836465849","https://openalex.org/W1901129140","https://openalex.org/W1980287119","https://openalex.org/W1987083649","https://openalex.org/W1994616650","https://openalex.org/W2097117768","https://openalex.org/W2103496339","https://openalex.org/W2146502635","https://openalex.org/W2156387975","https://openalex.org/W2194775991","https://openalex.org/W2589065377","https://openalex.org/W2618398196","https://openalex.org/W2623790698","https://openalex.org/W2625063094","https://openalex.org/W2766844050","https://openalex.org/W2783859821","https://openalex.org/W2785523195","https://openalex.org/W2808958252","https://openalex.org/W2886837499","https://openalex.org/W2894972989","https://openalex.org/W2899790086","https://openalex.org/W2900959181","https://openalex.org/W2908046483","https://openalex.org/W2952574409","https://openalex.org/W2962698540","https://openalex.org/W2962726660","https://openalex.org/W2962767131","https://openalex.org/W2963052924","https://openalex.org/W2963285578","https://openalex.org/W2963446712","https://openalex.org/W2963519230","https://openalex.org/W2963563140","https://openalex.org/W2963731874","https://openalex.org/W2964068654","https://openalex.org/W2964121744","https://openalex.org/W3118608800","https://openalex.org/W3141595720","https://openalex.org/W4293455017","https://openalex.org/W4297779039","https://openalex.org/W4297790878","https://openalex.org/W6631190155","https://openalex.org/W6637242042","https://openalex.org/W6638667902","https://openalex.org/W6639824700","https://openalex.org/W6681435938","https://openalex.org/W6682889407","https://openalex.org/W6685562342","https://openalex.org/W6687483927","https://openalex.org/W6717522514","https://openalex.org/W6725739302","https://openalex.org/W6731278065","https://openalex.org/W6738373677","https://openalex.org/W6738897456","https://openalex.org/W6739166439","https://openalex.org/W6746154026","https://openalex.org/W6747620207","https://openalex.org/W6747969630","https://openalex.org/W6751923770","https://openalex.org/W6752544856","https://openalex.org/W6753892653","https://openalex.org/W6754840104","https://openalex.org/W6755310813","https://openalex.org/W6756001544","https://openalex.org/W6757928074","https://openalex.org/W6780493881","https://openalex.org/W6787972765"],"related_works":["https://openalex.org/W2591697403","https://openalex.org/W2944728705","https://openalex.org/W2904022177","https://openalex.org/W2359348847","https://openalex.org/W3011538607","https://openalex.org/W4287755480","https://openalex.org/W3113607506","https://openalex.org/W4297779039","https://openalex.org/W3172664294","https://openalex.org/W4220659530"],"abstract_inverted_index":{"Modern":[0],"deep":[1],"neural":[2],"network":[3],"(DNN)":[4],"trainings":[5,30,62,112,148],"utilize":[6],"various":[7],"training":[8,47,116,151],"techniques,":[9],"e.g.,":[10],"nonlinear":[11],"activation":[12],"functions,":[13],"batch":[14],"normalization,":[15],"skip-connections,":[16],"etc.":[17],"Despite":[18],"their":[19],"effectiveness,":[20],"it":[21],"is":[22],"still":[23],"mysterious":[24],"how":[25],"they":[26],"help":[27],"accelerate":[28],"DNN":[29,50,61,111,147],"in":[31,95],"practice.":[32],"In":[33],"this":[34],"paper,":[35],"we":[36,53,83,108],"provide":[37],"an":[38],"empirical":[39],"study":[40],"of":[41,45,59],"the":[42,56,71,79,99,115,124,134,141,144,150,158,168,176],"regularization":[43,105,130,164],"effect":[44],"these":[46],"techniques":[48,117,152],"on":[49,103],"optimization.":[51],"Specifically,":[52],"find":[54,109],"that":[55,69,85,110,113,133,167],"optimization":[57,97],"trajectories":[58],"successful":[60],"consistently":[63],"obey":[64,123,157],"a":[65,87,92,104,119,128,162],"certain":[66],"regularity":[67,88,125,159],"principle":[68,89,126,160],"regularizes":[70],"model":[72,135,169],"update":[73],"direction":[74],"to":[75,91],"be":[76],"aligned":[77,139,174],"with":[78,127,140,161,175],"trajectory":[80],"direction.":[81],"Theoretically,":[82],"show":[84],"such":[86],"leads":[90],"convergence":[93,100,121,155],"guarantee":[94],"nonconvex":[96],"and":[98,122,156],"rate":[101],"depends":[102],"parameter.":[106],"Empirically,":[107],"apply":[114],"achieve":[118],"fast":[120],"large":[129],"parameter,":[131,165],"implying":[132,166],"updates":[136,170],"are":[137,171],"well":[138,173],"trajectory.":[142,177],"On":[143],"other":[145],"hand,":[146],"without":[149],"have":[153],"slow":[154],"small":[163],"not":[172]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}