{"id":"https://openalex.org/W2799640043","doi":"https://doi.org/10.1109/issre.2018.00021","title":"DeepMutation: Mutation Testing of Deep Learning Systems","display_name":"DeepMutation: Mutation Testing of Deep Learning Systems","publication_year":2018,"publication_date":"2018-10-01","ids":{"openalex":"https://openalex.org/W2799640043","doi":"https://doi.org/10.1109/issre.2018.00021","mag":"2799640043"},"language":"en","primary_location":{"id":"doi:10.1109/issre.2018.00021","is_oa":false,"landing_page_url":"https://doi.org/10.1109/issre.2018.00021","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE 29th International Symposium on Software Reliability Engineering (ISSRE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101468661","display_name":"Lei Ma","orcid":"https://orcid.org/0000-0002-8621-2420"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]},{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN","SG"],"is_corresponding":true,"raw_author_name":"Lei Ma","raw_affiliation_strings":["Harbin Inst. of Technol., Harbin, China","Nanyang Technological University, Singapore"],"affiliations":[{"raw_affiliation_string":"Harbin Inst. of Technol., Harbin, China","institution_ids":["https://openalex.org/I204983213"]},{"raw_affiliation_string":"Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101905312","display_name":"Fuyuan Zhang","orcid":"https://orcid.org/0009-0001-6560-5102"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Fuyuan Zhang","raw_affiliation_strings":["Nanyang Technol. Univ., Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"Nanyang Technol. Univ., Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102860446","display_name":"Jiyuan Sun","orcid":"https://orcid.org/0000-0002-6701-6598"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Jiyuan Sun","raw_affiliation_strings":["Kyushu Univ., Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu Univ., Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009850797","display_name":"Minhui Xue","orcid":"https://orcid.org/0000-0002-9172-4252"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Minhui Xue","raw_affiliation_strings":["Nanyang Technol. Univ., Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"Nanyang Technol. Univ., Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100374381","display_name":"Bo Li","orcid":"https://orcid.org/0000-0001-9407-9503"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bo Li","raw_affiliation_strings":["Univ. of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"Univ. of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034751153","display_name":"Felix Juefei-Xu","orcid":"https://orcid.org/0000-0002-0857-8611"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Felix Juefei-Xu","raw_affiliation_strings":["Carnegie Mellon Univ., Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon Univ., Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045231545","display_name":"Chao Xie","orcid":"https://orcid.org/0000-0001-7542-1270"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Chao Xie","raw_affiliation_strings":["Kyushu Univ., Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu Univ., Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106407830","display_name":"Li Li","orcid":"https://orcid.org/0000-0003-2990-1614"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Li Li","raw_affiliation_strings":["Monash Univ., Melbourne, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"Monash Univ., Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100355692","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0001-7300-9215"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["Nanyang Technol. Univ., Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"Nanyang Technol. Univ., Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065190767","display_name":"Jianjun Zhao","orcid":"https://orcid.org/0000-0001-8083-4352"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Jianjun Zhao","raw_affiliation_strings":["Kyushu Univ., Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu Univ., Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100392449","display_name":"Yadong Wang","orcid":"https://orcid.org/0000-0001-9627-4492"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yadong Wang","raw_affiliation_strings":["Harbin Inst. of Technol., Harbin, China"],"affiliations":[{"raw_affiliation_string":"Harbin Inst. of Technol., Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5101468661"],"corresponding_institution_ids":["https://openalex.org/I172675005","https://openalex.org/I204983213"],"apc_list":null,"apc_paid":null,"fwci":53.8698,"has_fulltext":false,"cited_by_count":364,"citation_normalized_percentile":{"value":0.99879735,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"100","last_page":"111"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.7602394223213196},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6979122161865234},{"id":"https://openalex.org/keywords/mnist-database","display_name":"MNIST database","score":0.6689062118530273},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6101574897766113},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.58258056640625},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.5460862517356873},{"id":"https://openalex.org/keywords/regression-testing","display_name":"Regression testing","score":0.5029796957969666},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4805608093738556},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.4703928232192993},{"id":"https://openalex.org/keywords/robustness-testing","display_name":"Robustness testing","score":0.442778617143631},{"id":"https://openalex.org/keywords/test-data","display_name":"Test data","score":0.4362439215183258},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4192183315753937},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.4172937572002411},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.3982599377632141},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.3879382610321045},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.36052948236465454},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.2846812605857849},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.28320392966270447},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.18659287691116333},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.15584397315979004},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.1528448462486267},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"Regression analysis","score":0.1384001076221466}],"concepts":[{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.7602394223213196},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6979122161865234},{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.6689062118530273},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6101574897766113},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.58258056640625},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.5460862517356873},{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.5029796957969666},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4805608093738556},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.4703928232192993},{"id":"https://openalex.org/C137726913","wikidata":"https://www.wikidata.org/wiki/Q7353550","display_name":"Robustness testing","level":3,"score":0.442778617143631},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.4362439215183258},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4192183315753937},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.4172937572002411},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3982599377632141},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.3879382610321045},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.36052948236465454},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2846812605857849},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.28320392966270447},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.18659287691116333},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.15584397315979004},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.1528448462486267},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.1384001076221466},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C58166","wikidata":"https://www.wikidata.org/wiki/Q224821","display_name":"Fuzzy logic","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/issre.2018.00021","is_oa":false,"landing_page_url":"https://doi.org/10.1109/issre.2018.00021","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE 29th International Symposium on Software Reliability Engineering (ISSRE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":93,"referenced_works":["https://openalex.org/W76940708","https://openalex.org/W120538599","https://openalex.org/W127000942","https://openalex.org/W800369072","https://openalex.org/W1526416044","https://openalex.org/W1532772304","https://openalex.org/W1538643901","https://openalex.org/W1554663460","https://openalex.org/W1567503821","https://openalex.org/W1570448133","https://openalex.org/W1665214252","https://openalex.org/W1945616565","https://openalex.org/W1967137625","https://openalex.org/W1974858479","https://openalex.org/W1993760289","https://openalex.org/W1996596150","https://openalex.org/W2014011458","https://openalex.org/W2016053056","https://openalex.org/W2042243351","https://openalex.org/W2049695835","https://openalex.org/W2051948002","https://openalex.org/W2061227290","https://openalex.org/W2099845725","https://openalex.org/W2103484079","https://openalex.org/W2108557864","https://openalex.org/W2110664127","https://openalex.org/W2111068183","https://openalex.org/W2112796928","https://openalex.org/W2117539524","https://openalex.org/W2117812871","https://openalex.org/W2119021158","https://openalex.org/W2120552859","https://openalex.org/W2120990696","https://openalex.org/W2121213139","https://openalex.org/W2121615981","https://openalex.org/W2122556103","https://openalex.org/W2135841285","https://openalex.org/W2142557127","https://openalex.org/W2156723666","https://openalex.org/W2158769184","https://openalex.org/W2162406453","https://openalex.org/W2165073069","https://openalex.org/W2166285667","https://openalex.org/W2168231600","https://openalex.org/W2247455710","https://openalex.org/W2257979135","https://openalex.org/W2402144811","https://openalex.org/W2466388701","https://openalex.org/W2542010156","https://openalex.org/W2594877703","https://openalex.org/W2616028256","https://openalex.org/W2753704268","https://openalex.org/W2761709036","https://openalex.org/W2775273147","https://openalex.org/W2783555701","https://openalex.org/W2784570262","https://openalex.org/W2793633339","https://openalex.org/W2794609696","https://openalex.org/W2804337238","https://openalex.org/W2809182766","https://openalex.org/W2953384591","https://openalex.org/W2962824341","https://openalex.org/W2963207607","https://openalex.org/W2963208512","https://openalex.org/W2963327228","https://openalex.org/W2963448658","https://openalex.org/W2963564844","https://openalex.org/W2963857521","https://openalex.org/W2977882543","https://openalex.org/W3099444373","https://openalex.org/W3118608800","https://openalex.org/W3157610735","https://openalex.org/W3162679235","https://openalex.org/W4285719527","https://openalex.org/W4302294892","https://openalex.org/W4388297464","https://openalex.org/W6603073347","https://openalex.org/W6603111959","https://openalex.org/W6604879145","https://openalex.org/W6637242042","https://openalex.org/W6640425456","https://openalex.org/W6678317597","https://openalex.org/W6684859321","https://openalex.org/W6690879419","https://openalex.org/W6713134421","https://openalex.org/W6734921443","https://openalex.org/W6743814675","https://openalex.org/W6744800992","https://openalex.org/W6745566777","https://openalex.org/W6746425455","https://openalex.org/W6748052837","https://openalex.org/W6749550085","https://openalex.org/W6752716229"],"related_works":["https://openalex.org/W2388956185","https://openalex.org/W2023403851","https://openalex.org/W2538766212","https://openalex.org/W2028796071","https://openalex.org/W2795302276","https://openalex.org/W2052414005","https://openalex.org/W2187337904","https://openalex.org/W2098051004","https://openalex.org/W3139873062","https://openalex.org/W2607833707"],"abstract_inverted_index":{"Deep":[0],"learning":[1],"(DL)":[2],"defines":[3],"a":[4,34,80,94,130,162,185,199],"new":[5],"data-driven":[6],"programming":[7],"paradigm":[8],"where":[9],"the":[10,18,40,51,98,104,140,150,172,203,212,218,227],"internal":[11],"system":[12],"logic":[13],"is":[14,28,43,79,232],"largely":[15],"shaped":[16],"by":[17,148],"training":[19,177,180,200],"data.":[20,144],"The":[21,37,224],"standard":[22],"way":[23],"of":[24,39,44,50,86,142,153,164,174,187,205,226],"evaluating":[25],"DL":[26,59,123,136,175,196,244],"models":[27,60,197],"to":[29,47,91,103,122,138,168,171,215],"examine":[30],"their":[31],"performance":[32],"on":[33,214,234],"test":[35,41,57,65,87,95,143,206],"dataset.":[36],"quality":[38,84,141,204],"dataset":[42],"great":[45],"importance":[46],"gain":[48],"confidence":[49],"trained":[52],"models.":[53,245],"Using":[54],"an":[55],"inadequate":[56],"dataset,":[58],"that":[61,191],"have":[62],"achieved":[63],"high":[64],"accuracy":[66],"may":[67],"still":[68],"lack":[69],"generality":[70],"and":[71,110,179,240],"robustness.":[72],"In":[73,125],"traditional":[74,108,114,157],"software":[75,109],"testing,":[76],"mutation":[77,115,131,154,166,189,229],"testing":[78,116,132,155,230],"well-established":[81],"technique":[82],"for":[83,135],"evaluation":[85],"suites,":[88],"which":[89],"analyzes":[90],"what":[92,216],"extent":[93,217],"suite":[96],"detects":[97],"injected":[99,219],"faults.":[100],"However,":[101],"due":[102],"fundamental":[105],"difference":[106],"between":[107],"deep":[111],"learning-based":[112],"software,":[113,158],"techniques":[117,231],"cannot":[118],"be":[119,209,222],"directly":[120,192],"applied":[121],"systems.":[124],"this":[126],"paper,":[127],"we":[128,159,183],"propose":[129],"framework":[133],"specialized":[134],"systems":[137],"measure":[139],"To":[145],"do":[146],"this,":[147],"sharing":[149],"same":[151],"spirit":[152],"in":[156],"first":[160],"define":[161],"set":[163,186],"source-level":[165],"operators":[167,190],"inject":[169,193],"faults":[170,194,220],"source":[173],"(i.e.,":[176],"data":[178,207],"programs).":[181],"Then":[182],"design":[184],"model-level":[188],"into":[195],"without":[198],"process.":[201],"Eventually,":[202],"could":[208,221],"evaluated":[210],"from":[211],"analysis":[213],"detected.":[223],"usefulness":[225],"proposed":[228],"demonstrated":[233],"two":[235],"public":[236],"datasets,":[237],"namely":[238],"MNIST":[239],"CIFAR-10,":[241],"with":[242],"three":[243]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":34},{"year":2024,"cited_by_count":56},{"year":2023,"cited_by_count":43},{"year":2022,"cited_by_count":54},{"year":2021,"cited_by_count":68},{"year":2020,"cited_by_count":60},{"year":2019,"cited_by_count":40},{"year":2018,"cited_by_count":8}],"updated_date":"2026-04-05T06:14:27.290980","created_date":"2025-10-10T00:00:00"}