{"id":"https://openalex.org/W4311315393","doi":"https://doi.org/10.4114/intartif.vol25iss70pp95-109","title":"A Large-Scale Study of Activation Functions in Modern Deep Neural Network Architectures for Efficient Convergence","display_name":"A Large-Scale Study of Activation Functions in Modern Deep Neural Network Architectures for Efficient Convergence","publication_year":2022,"publication_date":"2022-12-08","ids":{"openalex":"https://openalex.org/W4311315393","doi":"https://doi.org/10.4114/intartif.vol25iss70pp95-109"},"language":"en","primary_location":{"id":"doi:10.4114/intartif.vol25iss70pp95-109","is_oa":true,"landing_page_url":"https://doi.org/10.4114/intartif.vol25iss70pp95-109","pdf_url":"https://journal.iberamia.org/index.php/intartif/article/download/845/174","source":{"id":"https://openalex.org/S4210203986","display_name":"INTELIGENCIA ARTIFICIAL","issn_l":"1137-3601","issn":["1137-3601","1988-3064"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310321681","host_organization_name":"Asociaci\u00f3n Espa\u00f1ola para la Inteligencia Artificial","host_organization_lineage":["https://openalex.org/P4310321681"],"host_organization_lineage_names":["Asociaci\u00f3n Espa\u00f1ola para la Inteligencia Artificial"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Inteligencia Artificial","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://journal.iberamia.org/index.php/intartif/article/download/845/174","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056877018","display_name":"Andrinandrasana David Rasamoelina","orcid":"https://orcid.org/0000-0002-4318-9507"},"institutions":[{"id":"https://openalex.org/I183764125","display_name":"Technical University of Ko\u0161ice","ror":"https://ror.org/05xm08015","country_code":"SK","type":"education","lineage":["https://openalex.org/I183764125"]}],"countries":["SK"],"is_corresponding":true,"raw_author_name":"Andrinandrasana David Rasamoelina","raw_affiliation_strings":["Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic"],"raw_orcid":"https://orcid.org/0000-0002-4318-9507","affiliations":[{"raw_affiliation_string":"Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic","institution_ids":["https://openalex.org/I183764125"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080406266","display_name":"Ivan \u010c\u00edk","orcid":null},"institutions":[{"id":"https://openalex.org/I183764125","display_name":"Technical University of Ko\u0161ice","ror":"https://ror.org/05xm08015","country_code":"SK","type":"education","lineage":["https://openalex.org/I183764125"]}],"countries":["SK"],"is_corresponding":false,"raw_author_name":"Ivan C\u00edk","raw_affiliation_strings":["Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic","institution_ids":["https://openalex.org/I183764125"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009231271","display_name":"Peter Sin\u010d\u00e1k","orcid":"https://orcid.org/0000-0002-0994-5495"},"institutions":[{"id":"https://openalex.org/I20230540","display_name":"University of Miskolc","ror":"https://ror.org/038g7dk46","country_code":"HU","type":"education","lineage":["https://openalex.org/I20230540"]}],"countries":["HU"],"is_corresponding":false,"raw_author_name":"Peter Sincak","raw_affiliation_strings":["Faculty of Mechanical Engineering and Informatics, University of Miskolc, Hungary"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Mechanical Engineering and Informatics, University of Miskolc, Hungary","institution_ids":["https://openalex.org/I20230540"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009091781","display_name":"Mari\u00e1n Mach","orcid":"https://orcid.org/0000-0001-7612-9204"},"institutions":[{"id":"https://openalex.org/I183764125","display_name":"Technical University of Ko\u0161ice","ror":"https://ror.org/05xm08015","country_code":"SK","type":"education","lineage":["https://openalex.org/I183764125"]}],"countries":["SK"],"is_corresponding":false,"raw_author_name":"Mari\u00e1n Mach","raw_affiliation_strings":["Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic","institution_ids":["https://openalex.org/I183764125"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061663526","display_name":"Luk\u00e1\u0161 Hru\u0161ka","orcid":null},"institutions":[{"id":"https://openalex.org/I183764125","display_name":"Technical University of Ko\u0161ice","ror":"https://ror.org/05xm08015","country_code":"SK","type":"education","lineage":["https://openalex.org/I183764125"]}],"countries":["SK"],"is_corresponding":false,"raw_author_name":"Luk\u00e1\u0161 Hru\u0161ka","raw_affiliation_strings":["Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Cybernetics and Artificial Intelligence, FEI TU of Kosice, Slovak Republic","institution_ids":["https://openalex.org/I183764125"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5056877018"],"corresponding_institution_ids":["https://openalex.org/I183764125"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.13656855,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"25","issue":"70","first_page":"95","last_page":"109"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9368000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.909500002861023,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/activation-function","display_name":"Activation function","score":0.88773113489151},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.6873195767402649},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6637864112854004},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6393139362335205},{"id":"https://openalex.org/keywords/parametric-statistics","display_name":"Parametric statistics","score":0.5919672250747681},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5402408242225647},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.5359230041503906},{"id":"https://openalex.org/keywords/residual-neural-network","display_name":"Residual neural network","score":0.5084890127182007},{"id":"https://openalex.org/keywords/quadratic-equation","display_name":"Quadratic equation","score":0.5029100775718689},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.46706515550613403},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.4434940218925476},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44197171926498413},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.19800567626953125},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.07637909054756165},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.06495758891105652}],"concepts":[{"id":"https://openalex.org/C38365724","wikidata":"https://www.wikidata.org/wiki/Q4677469","display_name":"Activation function","level":3,"score":0.88773113489151},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.6873195767402649},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6637864112854004},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6393139362335205},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.5919672250747681},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5402408242225647},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.5359230041503906},{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.5084890127182007},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.5029100775718689},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.46706515550613403},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4434940218925476},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44197171926498413},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19800567626953125},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.07637909054756165},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.06495758891105652},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.4114/intartif.vol25iss70pp95-109","is_oa":true,"landing_page_url":"https://doi.org/10.4114/intartif.vol25iss70pp95-109","pdf_url":"https://journal.iberamia.org/index.php/intartif/article/download/845/174","source":{"id":"https://openalex.org/S4210203986","display_name":"INTELIGENCIA ARTIFICIAL","issn_l":"1137-3601","issn":["1137-3601","1988-3064"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310321681","host_organization_name":"Asociaci\u00f3n Espa\u00f1ola para la Inteligencia Artificial","host_organization_lineage":["https://openalex.org/P4310321681"],"host_organization_lineage_names":["Asociaci\u00f3n Espa\u00f1ola para la Inteligencia Artificial"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Inteligencia Artificial","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:21c687e93f0c4ff0b10be6a35cf4e7fb","is_oa":true,"landing_page_url":"https://doaj.org/article/21c687e93f0c4ff0b10be6a35cf4e7fb","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Inteligencia Artificial, Vol 25, Iss 70 (2022)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.4114/intartif.vol25iss70pp95-109","is_oa":true,"landing_page_url":"https://doi.org/10.4114/intartif.vol25iss70pp95-109","pdf_url":"https://journal.iberamia.org/index.php/intartif/article/download/845/174","source":{"id":"https://openalex.org/S4210203986","display_name":"INTELIGENCIA ARTIFICIAL","issn_l":"1137-3601","issn":["1137-3601","1988-3064"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310321681","host_organization_name":"Asociaci\u00f3n Espa\u00f1ola para la Inteligencia Artificial","host_organization_lineage":["https://openalex.org/P4310321681"],"host_organization_lineage_names":["Asociaci\u00f3n Espa\u00f1ola para la Inteligencia Artificial"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Inteligencia Artificial","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4311315393.pdf","grobid_xml":"https://content.openalex.org/works/W4311315393.grobid-xml"},"referenced_works_count":62,"referenced_works":["https://openalex.org/W1211190365","https://openalex.org/W1522301498","https://openalex.org/W1576445103","https://openalex.org/W1665214252","https://openalex.org/W1677182931","https://openalex.org/W1686810756","https://openalex.org/W1690739335","https://openalex.org/W1921523184","https://openalex.org/W2035424729","https://openalex.org/W2108598243","https://openalex.org/W2112796928","https://openalex.org/W2156387975","https://openalex.org/W2163605009","https://openalex.org/W2168148636","https://openalex.org/W2194775991","https://openalex.org/W2549139847","https://openalex.org/W2607735137","https://openalex.org/W2622263826","https://openalex.org/W2626912289","https://openalex.org/W2752134738","https://openalex.org/W2777662428","https://openalex.org/W2783893534","https://openalex.org/W2796438033","https://openalex.org/W2797480241","https://openalex.org/W2861440650","https://openalex.org/W2886953980","https://openalex.org/W2892144052","https://openalex.org/W2902332991","https://openalex.org/W2910415896","https://openalex.org/W2912550626","https://openalex.org/W2922301133","https://openalex.org/W2949491641","https://openalex.org/W2949869682","https://openalex.org/W2953930489","https://openalex.org/W2955425717","https://openalex.org/W2960198392","https://openalex.org/W2962861284","https://openalex.org/W2963163009","https://openalex.org/W2963285578","https://openalex.org/W2963290108","https://openalex.org/W2963454111","https://openalex.org/W2963855133","https://openalex.org/W2963918968","https://openalex.org/W2969937244","https://openalex.org/W2971689018","https://openalex.org/W2979797179","https://openalex.org/W2999541016","https://openalex.org/W3004591935","https://openalex.org/W3034021666","https://openalex.org/W3101333873","https://openalex.org/W3101398262","https://openalex.org/W3118608800","https://openalex.org/W3125923133","https://openalex.org/W4233045210","https://openalex.org/W4239072543","https://openalex.org/W4288267703","https://openalex.org/W4297775537","https://openalex.org/W4297798488","https://openalex.org/W4300972885","https://openalex.org/W4309845474","https://openalex.org/W4320930577","https://openalex.org/W4394643672"],"related_works":["https://openalex.org/W3110577345","https://openalex.org/W2946235959","https://openalex.org/W4377865163","https://openalex.org/W4285161415","https://openalex.org/W3193857078","https://openalex.org/W2888956734","https://openalex.org/W3208304128","https://openalex.org/W2799384463","https://openalex.org/W4221145391","https://openalex.org/W3159619856"],"abstract_inverted_index":{"Activation":[0],"functions":[1,43,58,123],"play":[2],"an":[3],"important":[4],"role":[5],"in":[6,27,36,44,48],"the":[7,23,39,53,75,85,88,101,126],"convergence":[8],"of":[9,41,72,77,87,90,103],"learning":[10],"algorithms":[11],"based":[12,99],"on":[13,38,59,100],"neural":[14,17],"networks.":[15],"Theyprovide":[16],"networks":[18],"with":[19,94],"nonlinear":[20],"ability":[21],"and":[22,79,124],"possibility":[24],"to":[25],"fit":[26],"any":[28],"complex":[29],"data.":[30],"However,":[31],"no":[32],"deep":[33],"study":[34],"exists":[35],"theliterature":[37],"comportment":[40],"activation":[42,57,96,111,122],"modern":[45],"architecture.":[46],"Therefore,":[47],"this":[49],"research,":[50],"we":[51,83],"compare":[52],"18":[54],"most":[55],"used":[56],"multiple":[60],"datasets":[61],"(CIFAR-10,":[62],"CIFAR-100,":[63],"CALTECH-256)":[64],"using":[65,74],"4":[66],"different":[67,92],"models":[68],"(EfficientNet,ResNet,":[69],"a":[70,107],"variation":[71,117],"ResNet":[73],"bag":[76],"tricks,":[78],"MobileNet":[80],"V3).":[81],"Furthermore,":[82],"explore":[84],"shape":[86],"losslandscape":[89],"those":[91],"architectures":[93],"various":[95],"functions.":[97],"Lastly,":[98],"result":[102],"our":[104],"experimentation,we":[105],"introduce":[106],"new":[108],"locally":[109],"quadratic":[110],"function":[112],"namely":[113],"Hytana":[114,119],"alongside":[115],"one":[116],"Parametric":[118],"whichoutperforms":[120],"common":[121],"address":[125],"dying":[127],"ReLU":[128],"problem.":[129]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}