{"id":"https://openalex.org/W4285265112","doi":"https://doi.org/10.1109/tai.2022.3180272","title":"Delve Into Neural Activations: Toward Understanding Dying Neurons","display_name":"Delve Into Neural Activations: Toward Understanding Dying Neurons","publication_year":2022,"publication_date":"2022-06-09","ids":{"openalex":"https://openalex.org/W4285265112","doi":"https://doi.org/10.1109/tai.2022.3180272"},"language":"en","primary_location":{"id":"doi:10.1109/tai.2022.3180272","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2022.3180272","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008586572","display_name":"Ziping Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210131914","display_name":"Lancaster City Council","ror":"https://ror.org/0387ev146","country_code":"GB","type":"government","lineage":["https://openalex.org/I4210131914"]},{"id":"https://openalex.org/I67415387","display_name":"Lancaster University","ror":"https://ror.org/04f2nsd36","country_code":"GB","type":"education","lineage":["https://openalex.org/I67415387"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Ziping Jiang","raw_affiliation_strings":["LIRA Center, Lancaster University, Lancaster, U.K"],"affiliations":[{"raw_affiliation_string":"LIRA Center, Lancaster University, Lancaster, U.K","institution_ids":["https://openalex.org/I67415387","https://openalex.org/I4210131914"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100350307","display_name":"Yunpeng Wang","orcid":"https://orcid.org/0000-0001-8862-0009"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunpeng Wang","raw_affiliation_strings":["Huawei Technologies Co., Ltd, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Huawei Technologies Co., Ltd, Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075200053","display_name":"Chang\u2010Tsun Li","orcid":"https://orcid.org/0000-0003-4735-6138"},"institutions":[{"id":"https://openalex.org/I149704539","display_name":"Deakin University","ror":"https://ror.org/02czsnj07","country_code":"AU","type":"education","lineage":["https://openalex.org/I149704539"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Chang-Tsun Li","raw_affiliation_strings":["Deakin University, Waurn Ponds, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"Deakin University, Waurn Ponds, VIC, Australia","institution_ids":["https://openalex.org/I149704539"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039480864","display_name":"Plamen Angelov","orcid":"https://orcid.org/0000-0002-5770-934X"},"institutions":[{"id":"https://openalex.org/I4210131914","display_name":"Lancaster City Council","ror":"https://ror.org/0387ev146","country_code":"GB","type":"government","lineage":["https://openalex.org/I4210131914"]},{"id":"https://openalex.org/I67415387","display_name":"Lancaster University","ror":"https://ror.org/04f2nsd36","country_code":"GB","type":"education","lineage":["https://openalex.org/I67415387"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Plamen Angelov","raw_affiliation_strings":["LIRA Center, Lancaster University, Lancaster, U.K"],"affiliations":[{"raw_affiliation_string":"LIRA Center, Lancaster University, Lancaster, U.K","institution_ids":["https://openalex.org/I67415387","https://openalex.org/I4210131914"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083182498","display_name":"Richard Jiang","orcid":"https://orcid.org/0000-0003-1721-9474"},"institutions":[{"id":"https://openalex.org/I4210131914","display_name":"Lancaster City Council","ror":"https://ror.org/0387ev146","country_code":"GB","type":"government","lineage":["https://openalex.org/I4210131914"]},{"id":"https://openalex.org/I67415387","display_name":"Lancaster University","ror":"https://ror.org/04f2nsd36","country_code":"GB","type":"education","lineage":["https://openalex.org/I67415387"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Richard Jiang","raw_affiliation_strings":["LIRA Center, Lancaster University, Lancaster, U.K"],"affiliations":[{"raw_affiliation_string":"LIRA Center, Lancaster University, Lancaster, U.K","institution_ids":["https://openalex.org/I67415387","https://openalex.org/I4210131914"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5008586572"],"corresponding_institution_ids":["https://openalex.org/I4210131914","https://openalex.org/I67415387"],"apc_list":null,"apc_paid":null,"fwci":0.8329,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.77483687,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"4","issue":"4","first_page":"959","last_page":"971"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sigmoid-function","display_name":"Sigmoid function","score":0.6815899610519409},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6429147124290466},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6288380026817322},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5697710514068604},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.5540993809700012},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5297179222106934},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5107543468475342},{"id":"https://openalex.org/keywords/activation-function","display_name":"Activation function","score":0.47811761498451233},{"id":"https://openalex.org/keywords/net","display_name":"Net (polyhedron)","score":0.4425677955150604},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2166425883769989},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.08782488107681274},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.08153057098388672}],"concepts":[{"id":"https://openalex.org/C81388566","wikidata":"https://www.wikidata.org/wiki/Q526668","display_name":"Sigmoid function","level":3,"score":0.6815899610519409},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6429147124290466},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6288380026817322},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5697710514068604},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.5540993809700012},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5297179222106934},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5107543468475342},{"id":"https://openalex.org/C38365724","wikidata":"https://www.wikidata.org/wiki/Q4677469","display_name":"Activation function","level":3,"score":0.47811761498451233},{"id":"https://openalex.org/C14166107","wikidata":"https://www.wikidata.org/wiki/Q253829","display_name":"Net (polyhedron)","level":2,"score":0.4425677955150604},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2166425883769989},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.08782488107681274},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.08153057098388672},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tai.2022.3180272","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2022.3180272","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2418174657","display_name":null,"funder_award_id":"EP/P009727/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G2844685089","display_name":null,"funder_award_id":"EP/P009727/2","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G4910419985","display_name":null,"funder_award_id":"RF-2019-492","funder_id":"https://openalex.org/F4320319993","funder_display_name":"Leverhulme Trust"}],"funders":[{"id":"https://openalex.org/F4320319993","display_name":"Leverhulme Trust","ror":"https://ror.org/012mzw131"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W1677182931","https://openalex.org/W1686810756","https://openalex.org/W1899504021","https://openalex.org/W1921523184","https://openalex.org/W2076063813","https://openalex.org/W2099579348","https://openalex.org/W2137983211","https://openalex.org/W2279639021","https://openalex.org/W2511261638","https://openalex.org/W2618530766","https://openalex.org/W2899663614","https://openalex.org/W2922301133","https://openalex.org/W3105432754","https://openalex.org/W4297775537","https://openalex.org/W4312655849","https://openalex.org/W4320930577","https://openalex.org/W6631636882","https://openalex.org/W6631943919","https://openalex.org/W6637359451","https://openalex.org/W6637373629","https://openalex.org/W6638545294","https://openalex.org/W6638667902","https://openalex.org/W6639703010","https://openalex.org/W6640185926","https://openalex.org/W6645550903","https://openalex.org/W6682889407","https://openalex.org/W6683595889","https://openalex.org/W6685562342","https://openalex.org/W6695282890","https://openalex.org/W6695533872","https://openalex.org/W6737664043","https://openalex.org/W6739659843","https://openalex.org/W6741653254","https://openalex.org/W6743923790","https://openalex.org/W6746023985","https://openalex.org/W6748230616","https://openalex.org/W6755144094","https://openalex.org/W6755310813","https://openalex.org/W6755977528","https://openalex.org/W6758094429","https://openalex.org/W6758955634","https://openalex.org/W6760385162","https://openalex.org/W6763744544","https://openalex.org/W6779567887","https://openalex.org/W6780226713","https://openalex.org/W6948022635"],"related_works":["https://openalex.org/W3024979424","https://openalex.org/W4283785902","https://openalex.org/W2895192346","https://openalex.org/W3032499992","https://openalex.org/W2041004593","https://openalex.org/W3134817226","https://openalex.org/W3110577345","https://openalex.org/W3136579697","https://openalex.org/W2811324119","https://openalex.org/W4385451479"],"abstract_inverted_index":{"Theoretically,":[0],"a":[1,44,71,144],"deep":[2],"neuron":[3,84,91],"network":[4,35,129],"with":[5,21,58,120,130,177],"nonlinear":[6],"activation":[7,38,48,61],"is":[8,133,188,216],"able":[9],"to":[10,50,78,168,219],"approximate":[11],"any":[12,158,170],"function,":[13],"while":[14],"empirically":[15],"the":[16,19,31,34,52,56,80,90,95,107,115,125,128,154,165,175,182,191,198,202,207,210,220,228,231,241,247],"performance":[17,243],"of":[18,33,55,83,110,127,140,143,164,185,201,209,230,240,249],"model":[20,57,221,242],"different":[22,121,178],"activations":[23,132],"varies":[24],"widely.":[25],"In":[26,40],"this":[27],"work,":[28],"we":[29,42,204,225],"investigate":[30],"expressivity":[32,82,126,184],"from":[36],"an":[37,59,99,238],"perspective.":[39],"particular,":[41],"introduce":[43],"<italic":[45,74,101,211,232],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[46,75,102,212,233],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">generalized":[47],"region/pattern</i>":[49],"describe":[51],"functional":[53],"relationship":[54],"arbitrary":[60],"function":[62],"and":[63,197],"illustrate":[64],"its":[65],"fundamental":[66],"properties.":[67],"We":[68,97],"then":[69],"propose":[70],"metric":[72],"named":[73],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">pattern":[76],"similarity</i>":[77],"evaluate":[79],"practical":[81,183],"networks":[85,187],"regarding":[86],"datasets":[87],"based":[88],"on":[89],"level":[92],"reaction":[93],"toward":[94],"input.":[96],"find":[98],"undocumented":[100],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">dying":[103,213,234],"neuron</i>":[104,214,235],"issue":[105,215],"that":[106,124,181,206],"postactivation":[108,141],"value":[109],"most":[111,163],"neurons":[112,166],"remain":[113],"in":[114,153,173],"same":[116,155],"region":[117,156],"for":[118],"data":[119,176],"labels,":[122,179],"implying":[123],"certain":[131],"greatly":[134],"constrained.":[135],"For":[136],"instance,":[137],"around":[138],"80%":[139],"values":[142],"well-trained":[145],"Sigmoid":[146],"net":[147,150],"or":[148],"Tanh":[149],"are":[151],"clustered":[152],"given":[157],"test":[159,199],"sample.":[160],"This":[161],"means":[162],"fail":[167],"provide":[169],"useful":[171],"information":[172],"distinguishing":[174],"suggesting":[180],"those":[186],"far":[189],"below":[190],"theoretical.":[192],"By":[193],"evaluating":[194],"our":[195],"metrics":[196],"accuracy":[200],"model,":[203],"show":[205],"seriousness":[208],"highly":[217],"related":[218],"performance.":[222],"At":[223],"last,":[224],"also":[226],"discussed":[227],"cause":[229],"issue,":[236],"providing":[237],"explanation":[239],"gap":[244],"caused":[245],"by":[246],"choice":[248],"activation.":[250]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}