{"id":"https://openalex.org/W4391614764","doi":"https://doi.org/10.1145/3644388","title":"DeepGD: A Multi-Objective Black-Box Test Selection Approach for Deep Neural Networks","display_name":"DeepGD: A Multi-Objective Black-Box Test Selection Approach for Deep Neural Networks","publication_year":2024,"publication_date":"2024-02-07","ids":{"openalex":"https://openalex.org/W4391614764","doi":"https://doi.org/10.1145/3644388"},"language":"en","primary_location":{"id":"doi:10.1145/3644388","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3644388","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3644388","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3644388","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059591494","display_name":"Zohreh Aghababaeyan","orcid":"https://orcid.org/0000-0001-9375-4095"},"institutions":[{"id":"https://openalex.org/I153718931","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16","country_code":"CA","type":"education","lineage":["https://openalex.org/I153718931"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zohreh Aghababaeyan","raw_affiliation_strings":["University of Ottawa, Ottawa, Canada"],"raw_orcid":"https://orcid.org/0000-0001-9375-4095","affiliations":[{"raw_affiliation_string":"University of Ottawa, Ottawa, Canada","institution_ids":["https://openalex.org/I153718931"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076024236","display_name":"Manel Abdellatif","orcid":"https://orcid.org/0000-0002-8647-1676"},"institutions":[{"id":"https://openalex.org/I9736820","display_name":"\u00c9cole de Technologie Sup\u00e9rieure","ror":"https://ror.org/0020snb74","country_code":"CA","type":"education","lineage":["https://openalex.org/I49663120","https://openalex.org/I9736820"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Manel Abdellatif","raw_affiliation_strings":["\u00c9cole de technologie sup\u00e9rieure, Montreal, Canada"],"raw_orcid":"https://orcid.org/0000-0002-8647-1676","affiliations":[{"raw_affiliation_string":"\u00c9cole de technologie sup\u00e9rieure, Montreal, Canada","institution_ids":["https://openalex.org/I9736820"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072142047","display_name":"Mahboubeh Dadkhah","orcid":"https://orcid.org/0000-0002-0436-8369"},"institutions":[{"id":"https://openalex.org/I153718931","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16","country_code":"CA","type":"education","lineage":["https://openalex.org/I153718931"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mahboubeh Dadkhah","raw_affiliation_strings":["University of Ottawa, Ottawa, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0436-8369","affiliations":[{"raw_affiliation_string":"University of Ottawa, Ottawa, Canada","institution_ids":["https://openalex.org/I153718931"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078533117","display_name":"Lionel Briand","orcid":"https://orcid.org/0000-0002-1393-1010"},"institutions":[{"id":"https://openalex.org/I153718931","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16","country_code":"CA","type":"education","lineage":["https://openalex.org/I153718931"]},{"id":"https://openalex.org/I230495080","display_name":"University of Limerick","ror":"https://ror.org/00a0n9e72","country_code":"IE","type":"education","lineage":["https://openalex.org/I230495080"]}],"countries":["CA","IE"],"is_corresponding":false,"raw_author_name":"Lionel Briand","raw_affiliation_strings":["Lero SFI Centre on Software Research and University of Limerick, Limerick, Ireland","University of Ottawa, Ottawa, Canada","University of Ottawa, Ottawa, Canada and Lero Centre, University of Limerick, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-1393-1010","affiliations":[{"raw_affiliation_string":"Lero SFI Centre on Software Research and University of Limerick, Limerick, Ireland","institution_ids":["https://openalex.org/I230495080"]},{"raw_affiliation_string":"University of Ottawa, Ottawa, Canada","institution_ids":["https://openalex.org/I153718931"]},{"raw_affiliation_string":"University of Ottawa, Ottawa, Canada and Lero Centre, University of Limerick, Ireland","institution_ids":["https://openalex.org/I230495080","https://openalex.org/I153718931"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.8508,"has_fulltext":true,"cited_by_count":27,"citation_normalized_percentile":{"value":0.98533724,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"33","issue":"6","first_page":"1","last_page":"29"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8478084802627563},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5798546671867371},{"id":"https://openalex.org/keywords/black-box","display_name":"Black box","score":0.5760550498962402},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5745766162872314},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5373104810714722},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.4914747476577759},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4726509153842926},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.42159903049468994}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8478084802627563},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5798546671867371},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.5760550498962402},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5745766162872314},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5373104810714722},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4914747476577759},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4726509153842926},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.42159903049468994},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3644388","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3644388","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3644388","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},{"id":"pmh:oai:espace2.etsmtl.ca:29018","is_oa":false,"landing_page_url":"https://espace2.etsmtl.ca/id/eprint/29018/","pdf_url":null,"source":{"id":"https://openalex.org/S4306402392","display_name":"Espace \u00c9TS (ETS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1341030882","host_organization_name":"Educational Testing Service","host_organization_lineage":["https://openalex.org/I1341030882"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article publi\u00e9 dans une revue, r\u00e9vis\u00e9 par les pairs"}],"best_oa_location":{"id":"doi:10.1145/3644388","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3644388","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3644388","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.4099999964237213,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G4863802590","display_name":null,"funder_award_id":"957254","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391614764.pdf","grobid_xml":"https://content.openalex.org/works/W4391614764.grobid-xml"},"referenced_works_count":74,"referenced_works":["https://openalex.org/W1506010655","https://openalex.org/W1599808047","https://openalex.org/W1609394770","https://openalex.org/W1686810756","https://openalex.org/W1719186128","https://openalex.org/W1867551913","https://openalex.org/W1993131108","https://openalex.org/W2007339694","https://openalex.org/W2052671997","https://openalex.org/W2065399007","https://openalex.org/W2073125568","https://openalex.org/W2098639318","https://openalex.org/W2108598243","https://openalex.org/W2115857089","https://openalex.org/W2126105956","https://openalex.org/W2127955369","https://openalex.org/W2138779671","https://openalex.org/W2151666086","https://openalex.org/W2242274110","https://openalex.org/W2486987313","https://openalex.org/W2592929672","https://openalex.org/W2601243251","https://openalex.org/W2616028256","https://openalex.org/W2792207129","https://openalex.org/W2793633339","https://openalex.org/W2811120218","https://openalex.org/W2911624154","https://openalex.org/W2946466522","https://openalex.org/W2948254043","https://openalex.org/W2948367246","https://openalex.org/W2954629067","https://openalex.org/W2954678575","https://openalex.org/W2963327228","https://openalex.org/W2968774823","https://openalex.org/W2979485058","https://openalex.org/W2981207549","https://openalex.org/W2994987245","https://openalex.org/W2995486425","https://openalex.org/W3012393889","https://openalex.org/W3021613070","https://openalex.org/W3041012898","https://openalex.org/W3042703469","https://openalex.org/W3045086295","https://openalex.org/W3092616598","https://openalex.org/W3099444373","https://openalex.org/W3103014337","https://openalex.org/W3105249702","https://openalex.org/W3105347387","https://openalex.org/W3120991880","https://openalex.org/W3139294479","https://openalex.org/W3148921802","https://openalex.org/W3164255502","https://openalex.org/W3176989815","https://openalex.org/W3178701007","https://openalex.org/W3185425515","https://openalex.org/W4221057950","https://openalex.org/W4221142422","https://openalex.org/W4224214784","https://openalex.org/W4225410698","https://openalex.org/W4226014440","https://openalex.org/W4226216828","https://openalex.org/W4235667863","https://openalex.org/W4283011843","https://openalex.org/W4284702108","https://openalex.org/W4285734659","https://openalex.org/W4302028668","https://openalex.org/W4319663674","https://openalex.org/W4362611784","https://openalex.org/W4367016230","https://openalex.org/W4384345653","https://openalex.org/W4385680786","https://openalex.org/W6749550085","https://openalex.org/W6784755435","https://openalex.org/W6810626217"],"related_works":["https://openalex.org/W2047881532","https://openalex.org/W4312601715","https://openalex.org/W4377865163","https://openalex.org/W4392828243","https://openalex.org/W4312822655","https://openalex.org/W3157170264","https://openalex.org/W4298185893","https://openalex.org/W4402427143","https://openalex.org/W2896078964","https://openalex.org/W4281399026"],"abstract_inverted_index":{"Deep":[0],"neural":[1],"networks":[2],"(DNNs)":[3],"are":[4],"widely":[5,159],"used":[6,160],"in":[7,144,168,187],"various":[8],"application":[9],"domains":[10],"such":[11],"as":[12,129,133],"image":[13],"processing,":[14],"speech":[15],"recognition,":[16],"and":[17,32,162,192],"natural":[18],"language":[19],"processing.":[20],"However,":[21],"testing":[22,40],"DNN":[23,41,53,94,146,164,201],"models":[24,42,165],"may":[25],"be":[26],"challenging":[27],"due":[28],"to":[29,68,76,127,197,208],"the":[30,58,98,104,138,145,210],"complexity":[31],"size":[33],"of":[34,100,106,140,170,189],"their":[35],"input":[36],"domain.":[37],"In":[38,51,80],"particular,":[39],"often":[43,63],"requires":[44],"generating":[45],"or":[46],"exploring":[47],"large":[48,114],"unlabeled":[49,115],"datasets.":[50,116],"practice,":[52],"test":[54,70,90,107,121,184],"oracles,":[55],"which":[56],"identify":[57],"correct":[59],"outputs":[60],"for":[61,93,200],"inputs,":[62],"require":[64],"expensive":[65],"manual":[66],"effort":[67],"label":[69],"data,":[71],"possibly":[72],"involving":[73],"multiple":[74],"experts":[75],"ensure":[77],"labeling":[78,101],"correctness.":[79],"this":[81],"article,":[82],"we":[83],"propose":[84],"DeepGD":[85,117,180,194],",":[86],"a":[87],"black-box":[88,183],"multi-objective":[89],"selection":[91,105,185],"approach":[92],"models.":[95],"It":[96],"reduces":[97],"cost":[99],"by":[102,148],"prioritizing":[103],"inputs":[108,122,132,207],"with":[109,123],"high":[110,124],"fault-revealing":[111,171],"power":[112],"from":[113],"not":[118],"only":[119],"selects":[120],"uncertainty":[125],"scores":[126],"trigger":[128],"many":[130],"mispredicted":[131,151],"possible":[134],"but":[135],"also":[136,195],"maximizes":[137],"probability":[139],"revealing":[141],"distinct":[142],"faults":[143],"model":[147,202],"selecting":[149],"diverse":[150],"inputs.":[152],"The":[153],"experimental":[154],"results":[155],"conducted":[156],"on":[157],"four":[158],"datasets":[161],"five":[163],"show":[166],"that":[167],"terms":[169,188],"ability,":[172],"(1)":[173],"white-box,":[174],"coverage-based":[175],"approaches":[176,186],"fare":[177],"poorly,":[178],"(2)":[179],"outperforms":[181],"existing":[182],"fault":[190],"detection,":[191],"(3)":[193],"leads":[196],"better":[198],"guidance":[199],"retraining":[203],"when":[204],"using":[205],"selected":[206],"augment":[209],"training":[211],"set.":[212]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-02T09:51:11.867554","created_date":"2024-02-08T00:00:00"}