{"id":"https://openalex.org/W4403511313","doi":"https://doi.org/10.1109/tse.2024.3482984","title":"TEASMA: A Practical Methodology for Test Adequacy Assessment of Deep Neural Networks","display_name":"TEASMA: A Practical Methodology for Test Adequacy Assessment of Deep Neural Networks","publication_year":2024,"publication_date":"2024-10-17","ids":{"openalex":"https://openalex.org/W4403511313","doi":"https://doi.org/10.1109/tse.2024.3482984"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3482984","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3482984","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092657440","display_name":"Amin Abbasishahkoo","orcid":null},"institutions":[{"id":"https://openalex.org/I153718931","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16","country_code":"CA","type":"education","lineage":["https://openalex.org/I153718931"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Amin Abbasishahkoo","raw_affiliation_strings":["School of EECS, University of Ottawa, Ottawa, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0001-6854-5726","affiliations":[{"raw_affiliation_string":"School of EECS, University of Ottawa, Ottawa, ON, Canada","institution_ids":["https://openalex.org/I153718931"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072142047","display_name":"Mahboubeh Dadkhah","orcid":"https://orcid.org/0000-0002-0436-8369"},"institutions":[{"id":"https://openalex.org/I153718931","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16","country_code":"CA","type":"education","lineage":["https://openalex.org/I153718931"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mahboubeh Dadkhah","raw_affiliation_strings":["School of EECS, University of Ottawa, Ottawa, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0436-8369","affiliations":[{"raw_affiliation_string":"School of EECS, University of Ottawa, Ottawa, ON, Canada","institution_ids":["https://openalex.org/I153718931"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078533117","display_name":"Lionel Briand","orcid":"https://orcid.org/0000-0002-1393-1010"},"institutions":[{"id":"https://openalex.org/I153718931","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16","country_code":"CA","type":"education","lineage":["https://openalex.org/I153718931"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Lionel Briand","raw_affiliation_strings":["School of EECS, University of Ottawa, Ottawa, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0002-1393-1010","affiliations":[{"raw_affiliation_string":"School of EECS, University of Ottawa, Ottawa, ON, Canada","institution_ids":["https://openalex.org/I153718931"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079401267","display_name":"Dayi Lin","orcid":"https://orcid.org/0000-0002-4034-6650"},"institutions":[{"id":"https://openalex.org/I4210115038","display_name":"Huawei Technologies (Canada)","ror":"https://ror.org/026venb53","country_code":"CA","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210115038"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Dayi Lin","raw_affiliation_strings":["Huawei Canada, Kingston, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0002-4034-6650","affiliations":[{"raw_affiliation_string":"Huawei Canada, Kingston, ON, Canada","institution_ids":["https://openalex.org/I4210115038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6109,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.74551456,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"50","issue":"12","first_page":"3307","last_page":"3329"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8762000203132629,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8762000203132629,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10876","display_name":"Fault Detection and Control Systems","score":0.766700029373169,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.7502999901771545,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.822954535484314},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5710327625274658},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5240338444709778},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44190138578414917},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4206932485103607},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.3757359981536865},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3498424291610718},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08758503198623657}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.822954535484314},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5710327625274658},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5240338444709778},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44190138578414917},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4206932485103607},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3757359981536865},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3498424291610718},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08758503198623657},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2024.3482984","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3482984","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.44999998807907104,"id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G1482659561","display_name":null,"funder_award_id":"13/RC/2094-2","funder_id":"https://openalex.org/F4320320847","funder_display_name":"Science Foundation Ireland"}],"funders":[{"id":"https://openalex.org/F4320320847","display_name":"Science Foundation Ireland","ror":"https://ror.org/0271asj38"},{"id":"https://openalex.org/F4320321487","display_name":"Canadian Network for Research and Innovation in Machining Technology, Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"},{"id":"https://openalex.org/F4320322183","display_name":"Huawei Technologies","ror":"https://ror.org/00cmhce21"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W151377110","https://openalex.org/W1987971958","https://openalex.org/W1989171126","https://openalex.org/W2001610032","https://openalex.org/W2007339694","https://openalex.org/W2102742419","https://openalex.org/W2112796928","https://openalex.org/W2117539524","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2408186052","https://openalex.org/W2616028256","https://openalex.org/W2725449579","https://openalex.org/W2788962378","https://openalex.org/W2799640043","https://openalex.org/W2804337238","https://openalex.org/W2804935296","https://openalex.org/W2850992922","https://openalex.org/W2886698796","https://openalex.org/W2954629067","https://openalex.org/W2954678575","https://openalex.org/W2963308851","https://openalex.org/W2963327228","https://openalex.org/W2964164993","https://openalex.org/W2968594320","https://openalex.org/W2979485058","https://openalex.org/W3000442532","https://openalex.org/W3042703469","https://openalex.org/W3048160496","https://openalex.org/W3090423701","https://openalex.org/W3090643686","https://openalex.org/W3120991880","https://openalex.org/W3135464648","https://openalex.org/W3135939397","https://openalex.org/W3161493619","https://openalex.org/W3163879210","https://openalex.org/W3164255502","https://openalex.org/W3178469298","https://openalex.org/W3185950485","https://openalex.org/W3200467243","https://openalex.org/W4224214784","https://openalex.org/W4226216828","https://openalex.org/W4235667863","https://openalex.org/W4283830352","https://openalex.org/W4286485305","https://openalex.org/W4299559948","https://openalex.org/W4311159382","https://openalex.org/W4319663674","https://openalex.org/W4388596098","https://openalex.org/W4391277903","https://openalex.org/W4391614764","https://openalex.org/W6622938502","https://openalex.org/W6637373629","https://openalex.org/W6784755435","https://openalex.org/W6787972765"],"related_works":["https://openalex.org/W641782856","https://openalex.org/W2961085424","https://openalex.org/W2391251536","https://openalex.org/W2362198218","https://openalex.org/W1982750869","https://openalex.org/W4376453582","https://openalex.org/W2019521278","https://openalex.org/W1984922432","https://openalex.org/W2375008505","https://openalex.org/W2085756966"],"abstract_inverted_index":{"Successful":[0],"deployment":[1],"of":[2,24,71,98,142,148,266,285,317],"Deep":[3],"Neural":[4],"Networks":[5],"(DNNs),":[6],"particularly":[7],"in":[8,26,50],"safety-critical":[9],"systems,":[10],"requires":[11],"their":[12,58,77,205],"validation":[13],"with":[14,164,261,314],"an":[15,153],"adequate":[16],"test":[17,27,31,72,99,115,150,167,338],"set":[18,151,218],"to":[19,48,56,93,108,136,204,210,310,336],"ensure":[20],"a":[21,61,87,127,134,149,244,277,329],"sufficient":[22],"degree":[23],"confidence":[25],"outcomes.":[28],"Although":[29],"well-established":[30],"adequacy":[32,97,155,168],"assessment":[33],"techniques":[34],"from":[35,256],"traditional":[36],"software,":[37],"such":[38,240],"as":[39,241],"mutation":[40,191],"analysis":[41,300],"and":[42,74,84,89,117,182,233,251,259,270,274,289,301,312,319],"coverage":[43],"criteria,":[44],"have":[45],"been":[46],"adapted":[47],"DNNs":[49],"recent":[51],"years,":[52],"we":[53,82],"still":[54],"need":[55],"investigate":[57],"application":[59],"within":[60],"comprehensive":[62,88],"methodology":[63,91],"for":[64,101,268,272,332,340],"accurately":[65,94],"predicting":[66],"the":[67,96,120,143,196,211,215,249,304],"fault":[68],"detection":[69],"ability":[70],"sets":[73,100,239],"thus":[75,118,157],"assessing":[76],"adequacy.":[78],"In":[79,103],"this":[80],"paper,":[81],"propose":[83],"evaluate":[85],"<i>TEASMA</i>,":[86],"practical":[90],"designed":[92],"assess":[95],"DNNs.":[102],"practice,":[104],"<i>TEASMA</i>":[105,132,163,327],"allows":[106],"engineers":[107],"decide":[109],"whether":[110,335],"they":[111],"can":[112],"trust":[113,337],"high-accuracy":[114],"results":[116,324,339],"validate":[119],"DNN":[121,128,198,231,341],"before":[122],"its":[123,159],"deployment.":[124],"Based":[125],"on":[126,190,298],"model's":[129],"training":[130,217],"set,":[131],"provides":[133,328],"procedure":[135],"build":[137],"accurate":[138],"DNN-specific":[139],"prediction":[140],"models":[141,232],"Fault":[144],"Detection":[145],"Rate":[146],"(FDR)":[147],"using":[152],"existing":[154],"metric,":[156],"enabling":[158],"assessment.":[160],"We":[161,186],"evaluated":[162],"four":[165],"state-of-the-art":[166],"metrics:":[169],"Distance-based":[170],"Surprise":[171,175],"Coverage":[172,176,180],"(DSC),":[173],"Likelihood-based":[174],"(LSC),":[177],"Input":[178],"Distribution":[179],"(IDC),":[181],"Mutation":[183],"Score":[184],"(MS).":[185],"calculated":[187],"MS":[188,269],"based":[189],"operators":[192,212],"that":[193,213,326],"directly":[194],"modify":[195,214],"trained":[197],"model":[199],"(i.e.,":[200,221],"post-training":[201],"operators)":[202],"due":[203],"significant":[206],"computational":[207],"advantage":[208],"compared":[209,309],"DNN's":[216],"or":[219],"program":[220],"pre-training":[222],"operators).":[223],"Our":[224],"extensive":[225],"empirical":[226],"evaluation,":[227],"conducted":[228],"across":[229,293],"multiple":[230],"input":[234,238],"sets,":[235],"including":[236],"large":[237],"ImageNet,":[242],"reveals":[243],"strong":[245],"linear":[246],"correlation":[247],"between":[248,287],"predicted":[250,290],"actual":[252,288],"FDR":[253,291],"values":[254,265,292,316],"derived":[255],"MS,":[257,302],"DSC,":[258],"IDC,":[260,313],"minimum":[262],"<inline-formula><tex-math":[263],"notation=\"LaTeX\">$R^{2}$</tex-math></inline-formula>":[264],"0.94":[267],"0.90":[271],"DSC":[273,311],"IDC.":[275],"Furthermore,":[276],"low":[278],"average":[279],"Root":[280],"Mean":[281],"Square":[282],"Error":[283],"(RMSE)":[284],"9%":[286],"all":[294],"subjects,":[295],"when":[296,308],"relying":[297],"regression":[299],"demonstrates":[303],"latter's":[305],"superior":[306],"accuracy":[307],"RMSE":[315],"0.17":[318],"0.18,":[320],"respectively.":[321],"Overall,":[322],"these":[323],"suggest":[325],"reliable":[330],"basis":[331],"confidently":[333],"deciding":[334],"models.":[342]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2024-10-18T00:00:00"}