{"id":"https://openalex.org/W2885066784","doi":null,"title":"Gradient Acceleration in Activation Functions.","display_name":"Gradient Acceleration in Activation Functions.","publication_year":2018,"publication_date":"2018-09-27","ids":{"openalex":"https://openalex.org/W2885066784","mag":"2885066784"},"language":"en","primary_location":{"id":"mag:2885066784","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=B1M9FjC5FQ","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016861708","display_name":"Sangchul Hahn","orcid":"https://orcid.org/0000-0003-2428-9534","observed_orcids":["https://orcid.org/0000-0003-2428-9534"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sangchul Hahn","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-2428-9534","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5079848178","display_name":"Heeyoul Choi","orcid":"https://orcid.org/0000-0002-0855-8725","observed_orcids":["https://orcid.org/0000-0002-0855-8725"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Heeyoul Choi","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-0855-8725","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3626,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.64516518,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":95},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8144999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8144999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.09549999982118607,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.057500001043081284,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dropout","display_name":"dropout","score":0.906000018119812},{"id":"https://openalex.org/keywords/activation-functions","display_name":"activation functions","score":0.9620000123977661},{"id":"https://openalex.org/keywords/saturation-region","display_name":"saturation region","score":0.597000002861023},{"id":"https://openalex.org/keywords/co-adaptation","display_name":"co-adaptation","score":0.7350000143051147},{"id":"https://openalex.org/keywords/backpropagation","display_name":"backpropagation","score":0.7689999938011169},{"id":"https://openalex.org/keywords/gradient-flow","display_name":"gradient flow","score":0.5730000138282776},{"id":"https://openalex.org/keywords/overfitting","display_name":"overfitting","score":0.5299999713897705},{"id":"https://openalex.org/keywords/regularization","display_name":"regularization","score":0.4699999988079071},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.6570000052452087},{"id":"https://openalex.org/keywords/flat-minima","display_name":"flat minima","score":0.4410000145435333}],"concepts":[{"id":"https://openalex.org/C2776145597","wikidata":"https://www.wikidata.org/wiki/Q25339462","display_name":"Dropout (neural networks)","level":2,"score":0.890379786491394},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.862740159034729},{"id":"https://openalex.org/C38365724","wikidata":"https://www.wikidata.org/wiki/Q4677469","display_name":"Activation function","level":3,"score":0.741992175579071},{"id":"https://openalex.org/C155032097","wikidata":"https://www.wikidata.org/wiki/Q798503","display_name":"Backpropagation","level":3,"score":0.657801628112793},{"id":"https://openalex.org/C117896860","wikidata":"https://www.wikidata.org/wiki/Q11376","display_name":"Acceleration","level":2,"score":0.6341813802719116},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6310402154922485},{"id":"https://openalex.org/C9930424","wikidata":"https://www.wikidata.org/wiki/Q7426587","display_name":"Saturation (graph theory)","level":2,"score":0.6128367781639099},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5627195835113525},{"id":"https://openalex.org/C2778168010","wikidata":"https://www.wikidata.org/wiki/Q1499786","display_name":"Climb","level":2,"score":0.5621551275253296},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.4713824689388275},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.45314982533454895},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4165467619895935},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2996254861354828},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22922638058662415},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1668042242527008},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.1333349347114563},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.11752799153327942},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C74650414","wikidata":"https://www.wikidata.org/wiki/Q11397","display_name":"Classical mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2885066784","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=B1M9FjC5FQ","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W4919037","https://openalex.org/W582134693","https://openalex.org/W1686810756","https://openalex.org/W1904365287","https://openalex.org/W2020683423","https://openalex.org/W2025768430","https://openalex.org/W2076063813","https://openalex.org/W2095705004","https://openalex.org/W2111406701","https://openalex.org/W2112796928","https://openalex.org/W2136836265","https://openalex.org/W2143612262","https://openalex.org/W2194775991","https://openalex.org/W2292443655","https://openalex.org/W2292877769","https://openalex.org/W2335728318","https://openalex.org/W2912811302","https://openalex.org/W2949117887","https://openalex.org/W2951595529","https://openalex.org/W2964325005","https://openalex.org/W3118608800"],"related_works":["https://openalex.org/W2934921594","https://openalex.org/W35527955","https://openalex.org/W7095380654","https://openalex.org/W3196353935","https://openalex.org/W7099207585","https://openalex.org/W2095705004","https://openalex.org/W2183112036","https://openalex.org/W7097994560","https://openalex.org/W4416430485","https://openalex.org/W3106830411"],"abstract_inverted_index":{"As":[0],"one":[1,51],"of":[2,70,104,180],"standard":[3],"approaches":[4],"to":[5,14,18,79,96,142,151],"train":[6],"deep":[7],"neural":[8],"networks,":[9],"dropout":[10,27,59,72,88,181],"has":[11,28],"been":[12,29],"applied":[13],"regularize":[15],"large":[16],"models":[17],"avoid":[19],"overfitting,":[20],"and":[21,74,182],"the":[22,45,58,98,101,115,146,152,158,163,168,185],"improvement":[23],"in":[24,114,118,135,145],"performance":[25,192],"by":[26,108],"explained":[30,91],"as":[31,92],"avoiding":[32],"co-adaptation":[33,55],"between":[34,39],"nodes.":[35],"However,":[36],"when":[37],"correlations":[38],"nodes":[40],"are":[41],"compared":[42],"after":[43],"training":[44],"networks":[46],"with":[47,193],"or":[48],"without":[49],"dropout,":[50],"question":[52],"arises":[53],"if":[54],"avoidance":[56],"explains":[57],"effect":[60],"completely.":[61],"In":[62],"this":[63,122],"paper,":[64],"we":[65,85,124],"propose":[66,75,125],"an":[67,93],"additional":[68],"explanation":[69,179],"why":[71],"works":[73],"a":[76,126,172],"new":[77,127],"technique":[78,95,128,188],"design":[80],"better":[81],"activation":[82,106,130,136,153],"functions.":[83],"First,":[84],"show":[86],"that":[87,139,184],"can":[89,155],"be":[90],"optimization":[94],"push":[97],"input":[99,150],"towards":[100],"saturation":[102,116,147,159],"area":[103,117,160],"nonlinear":[105],"function":[107,137,154],"accelerating":[109],"gradient":[110,133],"information":[111],"flowing":[112],"even":[113,144],"backpropagation.":[119],"Based":[120],"on":[121,171],"explanation,":[123],"for":[129],"functions,":[131],"{\\em":[132],"acceleration":[134],"(GAAF)},":[138],"accelerates":[140],"gradients":[141],"flow":[143],"area.":[148],"Then,":[149],"climb":[156],"onto":[157],"which":[161],"makes":[162],"network":[164],"more":[165],"robust":[166],"because":[167],"model":[169],"converges":[170],"flat":[173],"region.":[174],"Experiment":[175],"results":[176],"support":[177],"our":[178],"confirm":[183],"proposed":[186],"GAAF":[187],"improves":[189],"image":[190],"classification":[191],"expected":[194],"properties.":[195]},"counts_by_year":[{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":2}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2018-08-22T00:00:00"}