{"id":"https://openalex.org/W2994987245","doi":"https://doi.org/10.1145/3368089.3409754","title":"Is neuron coverage a meaningful measure for testing deep neural networks?","display_name":"Is neuron coverage a meaningful measure for testing deep neural networks?","publication_year":2020,"publication_date":"2020-11-08","ids":{"openalex":"https://openalex.org/W2994987245","doi":"https://doi.org/10.1145/3368089.3409754","mag":"2994987245"},"language":"en","primary_location":{"id":"doi:10.1145/3368089.3409754","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3368089.3409754","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3368089.3409754","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3368089.3409754","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044654199","display_name":"Fabrice Harel-Canada","orcid":null},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Fabrice Harel-Canada","raw_affiliation_strings":["University of California at Los Angeles, USA"],"affiliations":[{"raw_affiliation_string":"University of California at Los Angeles, USA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100779977","display_name":"Lingxiao Wang","orcid":"https://orcid.org/0000-0003-3757-3403"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lingxiao Wang","raw_affiliation_strings":["University of California at Los Angeles, USA"],"affiliations":[{"raw_affiliation_string":"University of California at Los Angeles, USA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003747461","display_name":"Muhammad Ali Gulzar","orcid":"https://orcid.org/0000-0002-8007-8662"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Muhammad Ali Gulzar","raw_affiliation_strings":["Virgina Tech, USA"],"affiliations":[{"raw_affiliation_string":"Virgina Tech, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051448391","display_name":"Quanquan Gu","orcid":"https://orcid.org/0000-0001-9830-793X"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quanquan Gu","raw_affiliation_strings":["University of California at Los Angeles, USA"],"affiliations":[{"raw_affiliation_string":"University of California at Los Angeles, USA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049451101","display_name":"Miryung Kim","orcid":"https://orcid.org/0000-0003-3802-1512"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Miryung Kim","raw_affiliation_strings":["University of California at Los Angeles, USA"],"affiliations":[{"raw_affiliation_string":"University of California at Los Angeles, USA","institution_ids":["https://openalex.org/I161318765"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5044654199"],"corresponding_institution_ids":["https://openalex.org/I161318765"],"apc_list":null,"apc_paid":null,"fwci":12.3685,"has_fulltext":true,"cited_by_count":148,"citation_normalized_percentile":{"value":0.98910453,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"851","last_page":"862"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9708999991416931,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9559999704360962,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7896407842636108},{"id":"https://openalex.org/keywords/naturalness","display_name":"Naturalness","score":0.7282082438468933},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.6839377880096436},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.6105325818061829},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5822576880455017},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5793690085411072},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5400218963623047},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.5305814146995544},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.5210379958152771},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4959348142147064},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.47552111744880676},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.46550920605659485},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.46136537194252014},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.44971925020217896},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.32995548844337463},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.17821639776229858},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.1710335910320282}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7896407842636108},{"id":"https://openalex.org/C134537474","wikidata":"https://www.wikidata.org/wiki/Q17144832","display_name":"Naturalness","level":2,"score":0.7282082438468933},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.6839377880096436},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.6105325818061829},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5822576880455017},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5793690085411072},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5400218963623047},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.5305814146995544},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5210379958152771},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4959348142147064},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.47552111744880676},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.46550920605659485},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.46136537194252014},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.44971925020217896},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.32995548844337463},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.17821639776229858},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.1710335910320282},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3368089.3409754","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3368089.3409754","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3368089.3409754","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:escholarship.org/ark:/13030/qt3c8107x5","is_oa":false,"landing_page_url":"https://escholarship.org/uc/item/3c8107x5","pdf_url":null,"source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"etd"},{"id":"mag:3160236325","is_oa":false,"landing_page_url":"https://jglobal.jst.go.jp/en/detail?JGLOBAL_ID=202002218045837514","pdf_url":null,"source":{"id":"https://openalex.org/S4306500161","display_name":"ACM Proceedings","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":"ACM Proceedings","raw_type":null}],"best_oa_location":{"id":"doi:10.1145/3368089.3409754","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3368089.3409754","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3368089.3409754","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2637195115","display_name":null,"funder_award_id":"4-18-1-","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G2709462027","display_name":null,"funder_award_id":"N00014-18-1-2037","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G3143604296","display_name":null,"funder_award_id":"CCF-1527923","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3400546487","display_name":null,"funder_award_id":"1764077","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3911675066","display_name":"SaTC: CORE: Small: Multi-Party High-dimensional Machine Learning with Privacy","funder_award_id":"1717950","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5103663181","display_name":null,"funder_award_id":"SaTC-1717950","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5679969530","display_name":null,"funder_award_id":"1723773","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6535578548","display_name":null,"funder_award_id":"CCF-1723773","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320308269","display_name":"Alexander von Humboldt-Stiftung","ror":"https://ror.org/012kf4317"},{"id":"https://openalex.org/F4320332195","display_name":"Samsung","ror":"https://ror.org/04w3jy968"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2994987245.pdf","grobid_xml":"https://content.openalex.org/works/W2994987245.grobid-xml"},"referenced_works_count":50,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1538643901","https://openalex.org/W1556550707","https://openalex.org/W1825675169","https://openalex.org/W1834627138","https://openalex.org/W1965555277","https://openalex.org/W1966913051","https://openalex.org/W1981970518","https://openalex.org/W1984284869","https://openalex.org/W1990414292","https://openalex.org/W1995875735","https://openalex.org/W2018093805","https://openalex.org/W2099471712","https://openalex.org/W2106072155","https://openalex.org/W2108598243","https://openalex.org/W2160815625","https://openalex.org/W2163246860","https://openalex.org/W2163605009","https://openalex.org/W2180612164","https://openalex.org/W2187250581","https://openalex.org/W2253993278","https://openalex.org/W2257979135","https://openalex.org/W2324595780","https://openalex.org/W2335728318","https://openalex.org/W2432004435","https://openalex.org/W2511730936","https://openalex.org/W2516574342","https://openalex.org/W2616028256","https://openalex.org/W2730550703","https://openalex.org/W2804337238","https://openalex.org/W2888233014","https://openalex.org/W2888307014","https://openalex.org/W2896500468","https://openalex.org/W2942630857","https://openalex.org/W2943040289","https://openalex.org/W2949605076","https://openalex.org/W2949650786","https://openalex.org/W2954629067","https://openalex.org/W2954678575","https://openalex.org/W2955224284","https://openalex.org/W2963207607","https://openalex.org/W2963327228","https://openalex.org/W2964153729","https://openalex.org/W2964253222","https://openalex.org/W2964308564","https://openalex.org/W2978725006","https://openalex.org/W3099444373","https://openalex.org/W3103148066","https://openalex.org/W4229494842","https://openalex.org/W4237293343"],"related_works":["https://openalex.org/W4295918990","https://openalex.org/W4313447549","https://openalex.org/W2028796071","https://openalex.org/W2410893741","https://openalex.org/W2067499928","https://openalex.org/W1888619389","https://openalex.org/W1968494916","https://openalex.org/W4376482197","https://openalex.org/W2794522096","https://openalex.org/W3022870375"],"abstract_inverted_index":{"Recent":[0],"effort":[1],"to":[2,56,87,109,114,127,137],"test":[3,13,51,60,93,131,175],"deep":[4,167],"learning":[5],"systems":[6],"has":[7],"produced":[8],"an":[9,54,129],"intuitive":[10],"and":[11,38,105,144,170,183],"compelling":[12],"criterion":[14],"called":[15],"neuron":[16,134,155],"coverage":[17,135,156],"(NC),":[18],"which":[19],"resembles":[20],"the":[21,29,47],"notion":[22],"of":[23,31,49],"traditional":[24],"code":[25],"coverage.":[26],"NC":[27,45,89,122],"measures":[28],"proportion":[30],"neurons":[32],"activated":[33],"in":[34,186],"a":[35,50,59,67,92,160,173],"neural":[36,168],"network":[37],"it":[39,125],"is":[40,107],"implicitly":[41],"assumed":[42],"that":[43,62,72,95,120,153,178],"increasing":[44,121,154],"improves":[46],"quality":[48],"suite.":[52],"In":[53],"attempt":[55],"automatically":[57],"generate":[58,91,128],"suite":[61,94],"increases":[63],"NC,":[64],"we":[65],"design":[66],"novel":[68],"diversity":[69],"promoting":[70],"regularizer":[71],"can":[73],"be":[74,159],"plugged":[75],"into":[76],"existing":[77],"adversarial":[78,98],"attack":[79],"algorithms.":[80],"We":[81],"then":[82],"assess":[83],"whether":[84],"such":[85],"attempts":[86],"increase":[88],"could":[90],"(1)":[96],"detects":[97],"attacks":[99],"successfully,":[100],"(2)":[101],"produces":[102],"natural":[103,142],"inputs,":[104,143],"(3)":[106],"unbiased":[108],"particular":[110],"class":[111],"predictions.":[112],"Contrary":[113],"expectation,":[115],"our":[116],"extensive":[117],"evaluation":[118],"finds":[119],"actually":[123],"makes":[124],"harder":[126],"effective":[130],"suite:":[132],"higher":[133],"leads":[136],"fewer":[138],"defects":[139],"detected,":[140],"less":[141],"more":[145],"biased":[146],"prediction":[147],"preferences.":[148],"Our":[149],"results":[150],"invoke":[151],"skepticism":[152],"may":[157],"not":[158],"meaningful":[161],"objective":[162],"for":[163,166,172],"generating":[164],"tests":[165],"networks":[169],"call":[171],"new":[174],"generation":[176],"technique":[177],"considers":[179],"defect":[180],"detection,":[181],"naturalness,":[182],"output":[184],"impartiality":[185],"tandem.":[187]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":40},{"year":2023,"cited_by_count":34},{"year":2022,"cited_by_count":31},{"year":2021,"cited_by_count":24},{"year":2020,"cited_by_count":2}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}