{"id":"https://openalex.org/W4382618754","doi":"https://doi.org/10.1145/3583133.3596321","title":"Empirical Loss Landscape Analysis of Neural Network Activation Functions","display_name":"Empirical Loss Landscape Analysis of Neural Network Activation Functions","publication_year":2023,"publication_date":"2023-07-15","ids":{"openalex":"https://openalex.org/W4382618754","doi":"https://doi.org/10.1145/3583133.3596321"},"language":"en","primary_location":{"id":"doi:10.1145/3583133.3596321","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3583133.3596321","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3583133.3596321","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Companion Conference on Genetic and Evolutionary Computation","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3583133.3596321","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037780608","display_name":"Anna Sergeevna Bosman","orcid":"https://orcid.org/0000-0003-3546-1467"},"institutions":[{"id":"https://openalex.org/I69552723","display_name":"University of Pretoria","ror":"https://ror.org/00g0p6g84","country_code":"ZA","type":"education","lineage":["https://openalex.org/I69552723"]}],"countries":["ZA"],"is_corresponding":true,"raw_author_name":"Anna Sergeevna Bosman","raw_affiliation_strings":["Computer Science, University of Pretoria, Pretoria, Gauteng, South Africa"],"affiliations":[{"raw_affiliation_string":"Computer Science, University of Pretoria, Pretoria, Gauteng, South Africa","institution_ids":["https://openalex.org/I69552723"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039197997","display_name":"Andries P. Engelbrecht","orcid":"https://orcid.org/0000-0002-0242-3539"},"institutions":[{"id":"https://openalex.org/I179311214","display_name":"Gulf University for Science & Technology","ror":"https://ror.org/04d9rzd67","country_code":"KW","type":"education","lineage":["https://openalex.org/I179311214"]},{"id":"https://openalex.org/I26092322","display_name":"Stellenbosch University","ror":"https://ror.org/05bk57929","country_code":"ZA","type":"education","lineage":["https://openalex.org/I26092322"]}],"countries":["KW","ZA"],"is_corresponding":false,"raw_author_name":"Andries Engelbrecht","raw_affiliation_strings":["Center for Applied Mathematics and Bioinformatics, Gulf University for Science and Technology, Kuwait City, Kuwait","University of Stellenbosch, Stellenbosch, South Africa","University of Stellenbosch, Stellenbosch, South Africa Center for Applied Mathematics and Bioinformatics, Gulf University for Science and Technology, Kuwait City, Kuwait"],"affiliations":[{"raw_affiliation_string":"Center for Applied Mathematics and Bioinformatics, Gulf University for Science and Technology, Kuwait City, Kuwait","institution_ids":["https://openalex.org/I179311214"]},{"raw_affiliation_string":"University of Stellenbosch, Stellenbosch, South Africa","institution_ids":["https://openalex.org/I26092322"]},{"raw_affiliation_string":"University of Stellenbosch, Stellenbosch, South Africa Center for Applied Mathematics and Bioinformatics, Gulf University for Science and Technology, Kuwait City, Kuwait","institution_ids":["https://openalex.org/I179311214","https://openalex.org/I26092322"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075927726","display_name":"Mard\u00e9 Helbig","orcid":"https://orcid.org/0000-0002-4907-0662"},"institutions":[{"id":"https://openalex.org/I11701301","display_name":"Griffith University","ror":"https://ror.org/02sc3r913","country_code":"AU","type":"education","lineage":["https://openalex.org/I11701301"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Marde Helbig","raw_affiliation_strings":["School of Information and Communication Technology, Griffith University, Southport, Australia"],"affiliations":[{"raw_affiliation_string":"School of Information and Communication Technology, Griffith University, Southport, Australia","institution_ids":["https://openalex.org/I11701301"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5037780608"],"corresponding_institution_ids":["https://openalex.org/I69552723"],"apc_list":null,"apc_paid":null,"fwci":0.6883,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.75083322,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"2029","last_page":"2037"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/activation-function","display_name":"Activation function","score":0.8320733308792114},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6905486583709717},{"id":"https://openalex.org/keywords/unit","display_name":"Unit (ring theory)","score":0.4928920269012451},{"id":"https://openalex.org/keywords/exponential-function","display_name":"Exponential function","score":0.4518997371196747},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.44009536504745483},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.43444326519966125},{"id":"https://openalex.org/keywords/linearity","display_name":"Linearity","score":0.41273146867752075},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.38801705837249756},{"id":"https://openalex.org/keywords/topology","display_name":"Topology (electrical circuits)","score":0.3773271143436432},{"id":"https://openalex.org/keywords/control-theory","display_name":"Control theory (sociology)","score":0.3734912872314453},{"id":"https://openalex.org/keywords/biological-system","display_name":"Biological system","score":0.33040812611579895},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.20151814818382263},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1830359399318695},{"id":"https://openalex.org/keywords/electronic-engineering","display_name":"Electronic engineering","score":0.1820373833179474},{"id":"https://openalex.org/keywords/mathematical-analysis","display_name":"Mathematical analysis","score":0.16738048195838928},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.09043225646018982}],"concepts":[{"id":"https://openalex.org/C38365724","wikidata":"https://www.wikidata.org/wiki/Q4677469","display_name":"Activation function","level":3,"score":0.8320733308792114},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6905486583709717},{"id":"https://openalex.org/C122637931","wikidata":"https://www.wikidata.org/wiki/Q118084","display_name":"Unit (ring theory)","level":2,"score":0.4928920269012451},{"id":"https://openalex.org/C151376022","wikidata":"https://www.wikidata.org/wiki/Q168698","display_name":"Exponential function","level":2,"score":0.4518997371196747},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.44009536504745483},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.43444326519966125},{"id":"https://openalex.org/C77170095","wikidata":"https://www.wikidata.org/wiki/Q1753188","display_name":"Linearity","level":2,"score":0.41273146867752075},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.38801705837249756},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.3773271143436432},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.3734912872314453},{"id":"https://openalex.org/C186060115","wikidata":"https://www.wikidata.org/wiki/Q30336093","display_name":"Biological system","level":1,"score":0.33040812611579895},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.20151814818382263},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1830359399318695},{"id":"https://openalex.org/C24326235","wikidata":"https://www.wikidata.org/wiki/Q126095","display_name":"Electronic engineering","level":1,"score":0.1820373833179474},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.16738048195838928},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.09043225646018982},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.0},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3583133.3596321","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3583133.3596321","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3583133.3596321","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Companion Conference on Genetic and Evolutionary Computation","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2306.16090","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.16090","pdf_url":"https://arxiv.org/pdf/2306.16090","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:research-repository.griffith.edu.au:10072/425863","is_oa":true,"landing_page_url":"http://hdl.handle.net/10072/425863","pdf_url":null,"source":{"id":"https://openalex.org/S4306402548","display_name":"Griffith Research Online (Griffith University, Queensland, Australia)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I11701301","host_organization_name":"Griffith University","host_organization_lineage":["https://openalex.org/I11701301"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference output"}],"best_oa_location":{"id":"doi:10.1145/3583133.3596321","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3583133.3596321","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3583133.3596321","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Companion Conference on Genetic and Evolutionary Computation","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.6899999976158142,"id":"https://metadata.un.org/sdg/11","display_name":"Sustainable cities and communities"}],"awards":[{"id":"https://openalex.org/G4457240904","display_name":null,"funder_award_id":"138194","funder_id":"https://openalex.org/F4320320671","funder_display_name":"National Research Foundation"},{"id":"https://openalex.org/G5244961615","display_name":null,"funder_award_id":"Thuthuka","funder_id":"https://openalex.org/F4320320671","funder_display_name":"National Research Foundation"},{"id":"https://openalex.org/G5941719640","display_name":null,"funder_award_id":"13819413","funder_id":"https://openalex.org/F4320320671","funder_display_name":"National Research Foundation"}],"funders":[{"id":"https://openalex.org/F4320320671","display_name":"National Research Foundation","ror":"https://ror.org/05s0g1g46"},{"id":"https://openalex.org/F4320334105","display_name":"Center for High Performance Computing","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4382618754.pdf","grobid_xml":"https://content.openalex.org/works/W4382618754.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W243319620","https://openalex.org/W246566237","https://openalex.org/W1533861849","https://openalex.org/W1548103535","https://openalex.org/W1599746992","https://openalex.org/W1665214252","https://openalex.org/W1921523184","https://openalex.org/W1975822826","https://openalex.org/W2001619934","https://openalex.org/W2028186927","https://openalex.org/W2030258362","https://openalex.org/W2044400415","https://openalex.org/W2069143585","https://openalex.org/W2107550850","https://openalex.org/W2137983211","https://openalex.org/W2156387975","https://openalex.org/W2219036994","https://openalex.org/W2240288625","https://openalex.org/W2287543196","https://openalex.org/W2588556732","https://openalex.org/W2766736793","https://openalex.org/W2777256551","https://openalex.org/W2787438119","https://openalex.org/W2789350251","https://openalex.org/W2791430930","https://openalex.org/W2795934185","https://openalex.org/W2797228403","https://openalex.org/W2854334010","https://openalex.org/W2898859254","https://openalex.org/W2963074136","https://openalex.org/W2963285578","https://openalex.org/W3011054152","https://openalex.org/W3089567359","https://openalex.org/W3123511746","https://openalex.org/W3134476582","https://openalex.org/W4294087498","https://openalex.org/W6632554308"],"related_works":["https://openalex.org/W3110577345","https://openalex.org/W2946235959","https://openalex.org/W2902935192","https://openalex.org/W3134817226","https://openalex.org/W4318066384","https://openalex.org/W3215912596","https://openalex.org/W3198895633","https://openalex.org/W4287019248","https://openalex.org/W2023858428","https://openalex.org/W4247324130"],"abstract_inverted_index":{"Activation":[0],"functions":[1,35],"play":[2],"a":[3],"significant":[4],"role":[5],"in":[6,108],"neural":[7,43,53],"network":[8,54,132],"design":[9],"by":[10],"enabling":[11],"non-linearity.":[12],"The":[13,101],"choice":[14],"of":[15,25,103],"activation":[16,34,68,116],"function":[17],"was":[18],"previously":[19],"shown":[20,74,87,123],"to":[21,75,88,96,124],"influence":[22],"the":[23,26,31,77,90,109,119],"properties":[24,39],"resulting":[27],"loss":[28,37,55,80,93,110],"landscape.":[29],"Understanding":[30],"relationship":[32],"between":[33],"and":[36,45,64,82,95,105,118,129],"landscape":[38,111],"is":[40,73,86,112],"important":[41],"for":[42,114],"architecture":[44],"training":[46],"algorithm":[47],"design.":[48],"This":[49],"study":[50],"empirically":[51],"investigates":[52],"landscapes":[56],"associated":[57],"with":[58,126],"hyperbolic":[59],"tangent,":[60],"rectified":[61],"linear":[62,66,71,84],"unit,":[63],"exponential":[65,83],"unit":[67,72,85],"functions.":[69],"Rectified":[70],"yield":[76,89],"most":[78],"convex":[79],"landscape,":[81,94],"least":[91],"flat":[92],"exhibit":[97],"superior":[98],"generalisation":[99],"performance.":[100],"presence":[102],"wide":[104],"narrow":[106,120],"valleys":[107,121],"established":[113],"all":[115],"functions,":[117],"are":[122],"correlate":[125],"saturated":[127],"neurons":[128],"implicitly":[130],"regularised":[131],"configurations.":[133]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3}],"updated_date":"2026-04-15T08:11:43.952461","created_date":"2025-10-10T00:00:00"}