{"id":"https://openalex.org/W2765833400","doi":null,"title":"Swish: a Self-Gated Activation Function","display_name":"Swish: a Self-Gated Activation Function","publication_year":2017,"publication_date":"2017-10-16","ids":{"openalex":"https://openalex.org/W2765833400","mag":"2765833400"},"language":"en","primary_location":{"id":"mag:2765833400","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1710.05941v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/1710.05941v1","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074823373","display_name":"Prajit Ramachandran","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Prajit Ramachandran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026064427","display_name":"Barret Zoph","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barret Zoph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5088551093","display_name":"Quoc Viet Le","orcid":"https://orcid.org/0000-0002-1087-2844","observed_orcids":["https://orcid.org/0000-0002-1087-2844"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Quoc V. Le","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-1087-2844","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":369,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5713000297546387,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5713000297546387,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.3716000020503998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.02070000022649765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/swish-activation-function","display_name":"Swish activation function","score":0.9739999771118164},{"id":"https://openalex.org/keywords/activation-functions","display_name":"activation functions","score":0.5720000267028809},{"id":"https://openalex.org/keywords/deep-neural-network","display_name":"deep neural network","score":0.7170000076293945},{"id":"https://openalex.org/keywords/training-dynamics","display_name":"training dynamics","score":0.777999997138977},{"id":"https://openalex.org/keywords/imagenet-classification","display_name":"ImageNet classification","score":0.746999979019165},{"id":"https://openalex.org/keywords/inceptionresnetv2","display_name":"InceptionResNetV2","score":0.9980000257492065}],"concepts":[{"id":"https://openalex.org/C38365724","wikidata":"https://www.wikidata.org/wiki/Q4677469","display_name":"Activation function","level":3,"score":0.8298271894454956},{"id":"https://openalex.org/C81388566","wikidata":"https://www.wikidata.org/wiki/Q526668","display_name":"Sigmoid function","level":3,"score":0.7392770051956177},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.6157934069633484},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5948436260223389},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5805067420005798},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5563524961471558},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4889637231826782},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4857192635536194},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3569062054157257},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.17512479424476624},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1027323305606842},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2765833400","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1710.05941v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:2765833400","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1710.05941v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W1572063013","https://openalex.org/W1677182931","https://openalex.org/W2064675550","https://openalex.org/W2117539524","https://openalex.org/W2159198041","https://openalex.org/W2176412452","https://openalex.org/W2194775991","https://openalex.org/W2302255633","https://openalex.org/W2433379750","https://openalex.org/W2546302380","https://openalex.org/W2553303224","https://openalex.org/W2569475573","https://openalex.org/W2612445135","https://openalex.org/W2619307294","https://openalex.org/W2626778328","https://openalex.org/W2949117887","https://openalex.org/W2963636093","https://openalex.org/W2964121744","https://openalex.org/W3118608800"],"related_works":["https://openalex.org/W2947682002","https://openalex.org/W2767286248","https://openalex.org/W2784978858","https://openalex.org/W4377024846","https://openalex.org/W4400611564","https://openalex.org/W3171742417","https://openalex.org/W4391594218","https://openalex.org/W4400377509","https://openalex.org/W4322631263","https://openalex.org/W2886690537"],"abstract_inverted_index":{"The":[0,113],"choice":[1],"of":[2,87,115],"activation":[3,25,58],"functions":[4],"in":[5,133],"deep":[6],"networks":[7],"has":[8],"a":[9,56,85],"significant":[10],"effect":[11],"on":[12,81,102],"the":[13,20,28],"training":[14],"dynamics":[15],"and":[16,23,109,117],"task":[17],"performance.":[18],"Currently,":[19],"most":[21],"successful":[22],"widely-used":[24],"function":[26],"is":[27,63],"Rectified":[29],"Linear":[30],"Unit":[31],"(ReLU).":[32],"Although":[33],"various":[34],"alternatives":[35],"to":[36,44,48,76,120,127],"ReLU":[37,80,121],"have":[38,42],"been":[39],"proposed,":[40],"none":[41],"managed":[43],"replace":[45,128],"it":[46,123],"due":[47],"inconsistent":[49],"gains.":[50],"In":[51],"this":[52],"work,":[53],"we":[54],"propose":[55],"new":[57],"function,":[59],"named":[60],"Swish,":[61],"which":[62],"simply":[64,92],"$f(x)":[65],"=":[66],"x":[67],"\\cdot":[68],"\\text{sigmoid}(x)$.":[69],"Our":[70],"experiments":[71],"show":[72],"that":[73],"Swish":[74,96,116,131],"tends":[75],"work":[77],"better":[78],"than":[79],"deeper":[82],"models":[83],"across":[84],"number":[86],"challenging":[88],"datasets.":[89],"For":[90],"example,":[91],"replacing":[93],"ReLUs":[94,129],"with":[95,130],"units":[97,132],"improves":[98],"top-1":[99],"classification":[100],"accuracy":[101],"ImageNet":[103],"by":[104],"0.9%":[105],"for":[106,111,125],"Mobile":[107],"NASNet-A":[108],"0.6%":[110],"Inception-ResNet-v2.":[112],"simplicity":[114],"its":[118],"similarity":[119],"make":[122],"easy":[124],"practitioners":[126],"any":[134],"neural":[135],"network.":[136]},"counts_by_year":[{"year":2025,"cited_by_count":37},{"year":2024,"cited_by_count":42},{"year":2023,"cited_by_count":54},{"year":2022,"cited_by_count":67},{"year":2021,"cited_by_count":73},{"year":2020,"cited_by_count":57},{"year":2019,"cited_by_count":22},{"year":2018,"cited_by_count":16},{"year":2017,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2017-11-10T00:00:00"}