{"id":"https://openalex.org/W3006821749","doi":"https://doi.org/10.1109/bigdata47090.2019.9005674","title":"On Neural Network Activation Functions and Optimizers in Relation to Polynomial Regression","display_name":"On Neural Network Activation Functions and Optimizers in Relation to Polynomial Regression","publication_year":2019,"publication_date":"2019-12-01","ids":{"openalex":"https://openalex.org/W3006821749","doi":"https://doi.org/10.1109/bigdata47090.2019.9005674","mag":"3006821749"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata47090.2019.9005674","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9005674","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084762762","display_name":"John Pomerat","orcid":null},"institutions":[{"id":"https://openalex.org/I83809506","display_name":"University of South Alabama","ror":"https://ror.org/01s7b5y08","country_code":"US","type":"education","lineage":["https://openalex.org/I83809506"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"John Pomerat","raw_affiliation_strings":["University of South Alabama, Mobile, Alabama"],"affiliations":[{"raw_affiliation_string":"University of South Alabama, Mobile, Alabama","institution_ids":["https://openalex.org/I83809506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059908930","display_name":"Aviv Segev","orcid":"https://orcid.org/0000-0002-9550-3895"},"institutions":[{"id":"https://openalex.org/I83809506","display_name":"University of South Alabama","ror":"https://ror.org/01s7b5y08","country_code":"US","type":"education","lineage":["https://openalex.org/I83809506"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aviv Segev","raw_affiliation_strings":["University of South Alabama, Mobile, Alabama"],"affiliations":[{"raw_affiliation_string":"University of South Alabama, Mobile, Alabama","institution_ids":["https://openalex.org/I83809506"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084806883","display_name":"Rituparna Datta","orcid":"https://orcid.org/0000-0003-3816-2438"},"institutions":[{"id":"https://openalex.org/I83809506","display_name":"University of South Alabama","ror":"https://ror.org/01s7b5y08","country_code":"US","type":"education","lineage":["https://openalex.org/I83809506"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rituparna Datta","raw_affiliation_strings":["University of South Alabama, Mobile, Alabama"],"affiliations":[{"raw_affiliation_string":"University of South Alabama, Mobile, Alabama","institution_ids":["https://openalex.org/I83809506"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5084762762"],"corresponding_institution_ids":["https://openalex.org/I83809506"],"apc_list":null,"apc_paid":null,"fwci":2.4532,"has_fulltext":false,"cited_by_count":47,"citation_normalized_percentile":{"value":0.91696227,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"2019","issue":null,"first_page":"6183","last_page":"6185"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/activation-function","display_name":"Activation function","score":0.9274320006370544},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.7302789688110352},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.658625066280365},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6105604767799377},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5392559766769409},{"id":"https://openalex.org/keywords/network-architecture","display_name":"Network architecture","score":0.46280190348625183},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.4609227180480957},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.43512290716171265},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.414571076631546},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35619205236434937},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.33677375316619873},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.2218647599220276}],"concepts":[{"id":"https://openalex.org/C38365724","wikidata":"https://www.wikidata.org/wiki/Q4677469","display_name":"Activation function","level":3,"score":0.9274320006370544},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.7302789688110352},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.658625066280365},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6105604767799377},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5392559766769409},{"id":"https://openalex.org/C193415008","wikidata":"https://www.wikidata.org/wiki/Q639681","display_name":"Network architecture","level":2,"score":0.46280190348625183},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.4609227180480957},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.43512290716171265},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.414571076631546},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35619205236434937},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.33677375316619873},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2218647599220276},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/bigdata47090.2019.9005674","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9005674","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},{"id":"mag:3087422017","is_oa":false,"landing_page_url":"https://jglobal.jst.go.jp/en/detail?JGLOBAL_ID=202002291513054733","pdf_url":null,"source":{"id":"https://openalex.org/S4306512817","display_name":"IEEE Conference Proceedings","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":"IEEE Conference Proceedings","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5799999833106995,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W1507342598","https://openalex.org/W1522301498","https://openalex.org/W1608227909","https://openalex.org/W2040870580","https://openalex.org/W2125818553","https://openalex.org/W2146502635","https://openalex.org/W2163605009","https://openalex.org/W2565516711","https://openalex.org/W2840432239","https://openalex.org/W2899675781","https://openalex.org/W2964121744","https://openalex.org/W6600284362","https://openalex.org/W6631190155","https://openalex.org/W6681435938","https://openalex.org/W6684191040","https://openalex.org/W6756026151"],"related_works":["https://openalex.org/W3110577345","https://openalex.org/W2946235959","https://openalex.org/W2902935192","https://openalex.org/W3134817226","https://openalex.org/W4318066384","https://openalex.org/W3215912596","https://openalex.org/W2032258609","https://openalex.org/W4234874385","https://openalex.org/W3198895633","https://openalex.org/W3016975909"],"abstract_inverted_index":{"Recently,":[0,48,322],"research":[1,50,66,221],"in":[2,173,328,409,495,516],"machine":[3,353,357],"learning":[4,32,341,358],"has":[5,55,62,171,244,343],"become":[6],"more":[7,400,471,514],"reliant":[8],"on":[9,51,67,120,428,480],"data-driven":[10],"approaches.":[11],"However,":[12],"understanding":[13,372],"the":[14,28,87,96,124,127,133,137,145,153,159,165,184,195,218,231,247,272,349,362,373,418,458,466,487,498,517,536],"general":[15,197],"theory":[16],"behind":[17],"optimal":[18,37,374],"neural":[19,34,38,52,70,97,331,382,403,425,443],"network":[20,39,53,71,332,383,404,426,444],"architecture":[21,40,54,72],"is,":[22],"arguably,":[23],"just":[24],"as":[25,356],"important.":[26],"With":[27],"proliferation":[29],"of":[30,65,89,110,126,139,186,205,217,220,240,306,330,352,364,388,420,424,434,457],"deep":[31,340],"and":[33,46,92,123,181,211,226,256,280,287,319,337,377,447,455,475,502],"networks,":[35],"finding":[36,521],"is":[41,116,269,384,391,470,513],"vital":[42],"for":[43,152,264,348,380,451,477,520],"both":[44],"accuracy":[45],"performance.":[47],"extensive":[49],"been":[56,63,246,326],"performed":[57],"[3],[4].":[58],"Additionally,":[59,508],"while":[60,449],"there":[61,303],"plenty":[64,305],"hidden":[68,452],"layer":[69,453],"[2],":[73],"activation":[74,84,111,129,149,198,206,224,234,375,396,445,468],"functions":[75,225,446],"are":[76,304],"often":[77],"not":[78],"considered.":[79],"In":[80,108,132,175,201,238,276,299,413],"a":[81,90,103,214,381,410,421],"network,":[82,98],"an":[83,140],"function":[85,105,130,150,199,235,376,469,501],"defines":[86],"output":[88],"neuron":[91],"introduces":[93],"non-linearities":[94],"into":[95,290],"enabling":[99],"it":[100],"to":[101,193,203,229,301,309,339,369,392,497],"be":[102,194,230],"universal":[104],"approximator":[106],"[12].":[107,200],"terms":[109,239,329],"functions,":[112,207,397],"one":[113,291],"significant":[114],"paper":[115,259,390],"Krizhevsky's":[117],"seminole":[118],"work":[119],"ImageNet":[121],"classification":[122],"creation":[125],"ReLU":[128,170,192,228,474],"[1].":[131,167],"paper,":[134,179,278,415],"Krizhevsky":[135],"outlines":[136],"construction":[138],"image":[141,161],"recognition":[142,162],"model":[143],"using":[144],"Rectified":[146],"Linear":[147],"Unit":[148],"(ReLU)":[151],"ImageNET":[154],"LSVRC-2010":[155],"competition":[156],"which":[157],"outperformed":[158],"state-of-the-art":[160],"systems":[163],"at":[164],"time":[166],"Since":[168],"then,":[169],"increased":[172],"popularity.":[174],"their":[176,277],"2018":[177],"conference":[178],"Bircano\u011flu":[180],"Arica,":[182],"with":[183,254],"assistance":[185],"231":[187],"distinct":[188],"training":[189,411],"procedures,":[190],"found":[191,227],"best":[196],"addition":[202,300],"comparisons":[204,394],"Nwankpa,":[208],"Ijomah,":[209],"Gachagan,":[210],"Marshall":[212],"conducted":[213],"meta":[215],"analysis":[216],"field":[219,351],"centered":[222],"around":[223],"most":[232,248],"popular":[233,249],"choice":[236,379],"[5].":[237],"optimizers,":[241,398],"gradient":[242,313],"descent":[243],"historically":[245],"loss":[250],"optimization":[251,289,500],"algorithm,":[252,292],"but":[253],"Kingma":[255,279],"Ba's":[257],"2014":[258],"[8],":[260],"Adam:":[261],"A":[262],"Method":[263],"Stochastic":[265,491],"Optimization,":[266],"Adam":[267,293,499,525],"optimizer":[268,378],"slowly":[270],"becoming":[271],"industry":[273],"standard":[274],"[11].":[275],"Ba":[281],"cleverly":[282],"combine":[283],"momentum":[284],"descent,":[285,314],"RMSprop,":[286],"Adagrad":[288,317],"(or":[294],"adaptive":[295],"moment":[296],"estimation)":[297],"[8].":[298],"Adam,":[302],"other":[307],"optimizers":[308,448,537],"choose":[310],"from,":[311],"including":[312],"RMSprop":[315],"[9],":[316],"[10],":[318],"Adadelta":[320],"[7].":[321],"many":[323,531],"breakthroughs":[324],"have":[325],"made":[327],"performance,":[333],"improved":[334],"GPU":[335],"performance":[336,419,533],"adaptation":[338],"tasks":[342,479],"created":[344],"massive":[345],"efficiency":[346,365],"increases":[347],"whole":[350],"learning.":[354],"Furthermore,":[355],"becomes":[359],"increasingly":[360],"optimized,":[361],"importance":[363],"improvements":[366],"will":[367],"continue":[368],"rise.":[370],"Thus,":[371],"relevant.":[385],"The":[386],"goal":[387],"this":[389,414],"make":[393],"between":[395,535],"and,":[399],"generally,":[401],"entire":[402],"architectures,":[405],"through":[406],"measured":[407],"error":[408],"environment.":[412],"we":[416,440,463,509],"examine":[417],"wide":[422],"variety":[423],"configurations":[427,454],"randomly":[429],"generated":[430],"polynomial":[431,460,482],"data":[432],"sets":[433],"fixed":[435],"degree.":[436],"To":[437],"do":[438],"this,":[439],"compare":[441],"various":[442],"controlling":[450],"degree":[456],"underlying":[459],"dataset.":[461],"Curiously,":[462],"find":[464],"that":[465,511],"Sigmoid":[467],"accurate":[472],"than":[473,524],"Tanh":[476],"regression":[478],"low-featured":[481],"data.":[483],"We":[484],"also":[485],"reach":[486],"same":[488],"conclusion":[489],"regarding":[490],"Gradient":[492],"Descent":[493],"(SGD)":[494],"comparison":[496],"Root":[503],"Mean":[504],"Square":[505],"Propagation":[506],"(RMSprop).":[507],"observe":[510],"SGD":[512],"efficient":[515],"short":[518],"term":[519],"local":[522],"minimums":[523],"or":[526],"RMSprop;":[527],"however,":[528],"after":[529],"sufficiently":[530],"epochs,":[532],"differences":[534],"vanished.":[538]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":4}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}