{"id":"https://openalex.org/W3111827872","doi":"https://doi.org/10.1145/3324884.3416667","title":"Revisiting the relationship between fault detection, test adequacy criteria, and test set size","display_name":"Revisiting the relationship between fault detection, test adequacy criteria, and test set size","publication_year":2020,"publication_date":"2020-12-21","ids":{"openalex":"https://openalex.org/W3111827872","doi":"https://doi.org/10.1145/3324884.3416667","mag":"3111827872"},"language":"en","primary_location":{"id":"doi:10.1145/3324884.3416667","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3324884.3416667","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://figshare.com/articles/conference_contribution/Revisiting_the_Relationship_Between_Fault_Detection_Test_Adequacy_Criteria_and_Test_Set_Size/24613371","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030806671","display_name":"Yiqun T. Chen","orcid":"https://orcid.org/0000-0002-4100-1507"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yiqun T. Chen","raw_affiliation_strings":["University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050311714","display_name":"Rahul Gopinath","orcid":"https://orcid.org/0000-0001-9953-0930"},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Rahul Gopinath","raw_affiliation_strings":["CISPA Helmholtz-Zentrum, Saarbr\u00fccken, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz-Zentrum, Saarbr\u00fccken, Germany","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059341142","display_name":"Anita Tadakamalla","orcid":"https://orcid.org/0000-0002-7567-0127"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anita Tadakamalla","raw_affiliation_strings":["George Mason University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"George Mason University","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032068969","display_name":"Michael D. Ernst","orcid":"https://orcid.org/0000-0001-9379-277X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael D. Ernst","raw_affiliation_strings":["University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034123811","display_name":"Reid Holmes","orcid":"https://orcid.org/0000-0003-4213-494X"},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Reid Holmes","raw_affiliation_strings":["University of British Columbia, Vancouver, BC, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of British Columbia, Vancouver, BC, Canada","institution_ids":["https://openalex.org/I141945490"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079261847","display_name":"Gordon Fraser","orcid":"https://orcid.org/0000-0002-4364-6595"},"institutions":[{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gordon Fraser","raw_affiliation_strings":["University of Passau, Passau, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037613705","display_name":"Paul Ammann","orcid":null},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Paul Ammann","raw_affiliation_strings":["George Mason University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"George Mason University","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088079823","display_name":"Ren\u00e9 Just","orcid":"https://orcid.org/0000-0002-5982-275X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ren\u00e9 Just","raw_affiliation_strings":["University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":10.902,"has_fulltext":false,"cited_by_count":48,"citation_normalized_percentile":{"value":0.98649907,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"237","last_page":"249"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.991100013256073,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5548810362815857},{"id":"https://openalex.org/keywords/automatic-test-pattern-generation","display_name":"Automatic test pattern generation","score":0.5228277444839478},{"id":"https://openalex.org/keywords/fault-detection-and-isolation","display_name":"Fault detection and isolation","score":0.5143924355506897},{"id":"https://openalex.org/keywords/test-set","display_name":"Test set","score":0.491489052772522},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.48642778396606445},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.48515066504478455},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.47245195508003235},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.453583300113678},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.4421706795692444},{"id":"https://openalex.org/keywords/random-testing","display_name":"Random testing","score":0.4132747948169708},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40070128440856934},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.30779144167900085},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.2951101064682007},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.29456186294555664},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.27714940905570984},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.15262874960899353},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"Regression analysis","score":0.12507832050323486}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5548810362815857},{"id":"https://openalex.org/C17626397","wikidata":"https://www.wikidata.org/wiki/Q837455","display_name":"Automatic test pattern generation","level":3,"score":0.5228277444839478},{"id":"https://openalex.org/C152745839","wikidata":"https://www.wikidata.org/wiki/Q5438153","display_name":"Fault detection and isolation","level":3,"score":0.5143924355506897},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.491489052772522},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.48642778396606445},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.48515066504478455},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.47245195508003235},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.453583300113678},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4421706795692444},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.4132747948169708},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40070128440856934},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30779144167900085},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2951101064682007},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29456186294555664},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.27714940905570984},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.15262874960899353},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.12507832050323486},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C172707124","wikidata":"https://www.wikidata.org/wiki/Q423488","display_name":"Actuator","level":2,"score":0.0},{"id":"https://openalex.org/C134146338","wikidata":"https://www.wikidata.org/wiki/Q1815901","display_name":"Electronic circuit","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3324884.3416667","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3324884.3416667","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:figshare.com:article/24613371","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/Revisiting_the_Relationship_Between_Fault_Detection_Test_Adequacy_Criteria_and_Test_Set_Size/24613371","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},{"id":"doi:10.60882/cispa.24613371.v1","is_oa":true,"landing_page_url":"https://doi.org/10.60882/cispa.24613371.v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050916","display_name":"CISPA Helmholtz Center","issn_l":null,"issn":[],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/24613371","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/Revisiting_the_Relationship_Between_Fault_Detection_Test_Adequacy_Criteria_and_Test_Set_Size/24613371","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.5899999737739563,"id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G1053651427","display_name":"CAREER: Toward Effective, Predictable, and Consistent Software Testing","funder_award_id":"1942055","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8161085601","display_name":"CRI: CI-EN: Collaborative Research: An Experimental Infrastructure and a Database of Real Faults to Foster Reproducibility in Software Engineering Research","funder_award_id":"1823172","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":65,"referenced_works":["https://openalex.org/W1492815834","https://openalex.org/W1495297318","https://openalex.org/W1511920456","https://openalex.org/W1715619412","https://openalex.org/W1968427676","https://openalex.org/W1971650562","https://openalex.org/W1976727279","https://openalex.org/W1985093013","https://openalex.org/W1992987499","https://openalex.org/W1996596150","https://openalex.org/W1998393968","https://openalex.org/W2008479535","https://openalex.org/W2013223191","https://openalex.org/W2014011458","https://openalex.org/W2016692667","https://openalex.org/W2024081693","https://openalex.org/W2048366343","https://openalex.org/W2049641724","https://openalex.org/W2050560030","https://openalex.org/W2063068372","https://openalex.org/W2066124661","https://openalex.org/W2075994652","https://openalex.org/W2100300194","https://openalex.org/W2102742419","https://openalex.org/W2104420598","https://openalex.org/W2106072155","https://openalex.org/W2107031757","https://openalex.org/W2107709519","https://openalex.org/W2107751709","https://openalex.org/W2113002312","https://openalex.org/W2113952909","https://openalex.org/W2126941999","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2138059712","https://openalex.org/W2145733341","https://openalex.org/W2149416862","https://openalex.org/W2150129373","https://openalex.org/W2155246571","https://openalex.org/W2156723666","https://openalex.org/W2166616527","https://openalex.org/W2170204382","https://openalex.org/W2172260321","https://openalex.org/W2174442795","https://openalex.org/W2278011425","https://openalex.org/W2334865765","https://openalex.org/W2395265379","https://openalex.org/W2504931705","https://openalex.org/W2518089210","https://openalex.org/W2546792321","https://openalex.org/W2786944871","https://openalex.org/W2788962378","https://openalex.org/W2790381016","https://openalex.org/W2865201172","https://openalex.org/W2884023932","https://openalex.org/W2949400639","https://openalex.org/W2966980041","https://openalex.org/W2973860448","https://openalex.org/W4212838134","https://openalex.org/W4236706032","https://openalex.org/W4251215559","https://openalex.org/W4256285596","https://openalex.org/W4292403327","https://openalex.org/W4298277474","https://openalex.org/W4300420355"],"related_works":["https://openalex.org/W3201070945","https://openalex.org/W2792951589","https://openalex.org/W2149949200","https://openalex.org/W146143826","https://openalex.org/W2107600598","https://openalex.org/W2913181721","https://openalex.org/W2950042520","https://openalex.org/W4301421709","https://openalex.org/W4307328323","https://openalex.org/W2134239377"],"abstract_inverted_index":{"The":[0],"research":[1],"community":[2],"has":[3],"long":[4],"recognized":[5],"a":[6,99,128,190,197,203],"complex":[7],"interrelationship":[8],"between":[9,71],"fault":[10,45,72,205],"detection,":[11,73],"test":[12,16,32,50,74,78,94,148,158,200,214],"adequacy":[13,40,51,75,149],"criteria,":[14,76],"and":[15,26,47,77,91,117,132,181,206],"set":[17,33,79,95,159,198],"size.":[18,80],"However,":[19],"there":[20],"is":[21,42,97],"substantial":[22],"confusion":[23],"about":[24],"whether":[25],"how":[27,37],"to":[28],"experimentally":[29,112],"control":[30],"for":[31,140,146,157,192,202,207],"size":[34,96,160],"when":[35,48],"assessing":[36,193],"well":[38,136],"an":[39,106,152],"criterion":[41],"correlated":[43],"with":[44,127],"detection":[46],"comparing":[49,147],"criteria.":[52],"Resolving":[53],"the":[54,59,69,85,120,174,194,209],"confusion,":[55],"this":[56,82,185],"paper":[57,83,186],"makes":[58],"following":[60],"contributions:":[61],"(1)":[62],"A":[63,144],"review":[64],"of":[65,68,88,119,123,130,176,196,199,212],"contradictory":[66],"analyses":[67],"relationships":[70],"Specifically,":[81],"addresses":[84],"supposed":[86],"contradiction":[87],"prior":[89,124],"work":[90],"explains":[92],"why":[93],"neither":[98],"confounding":[100],"variable,":[101],"as":[102,135,137],"previously":[103],"suggested,":[104],"nor":[105],"independent":[107],"variable":[108],"that":[109,172],"should":[110],"be":[111],"manipulated.":[113],"(2)":[114],"An":[115,169],"explication":[116],"discussion":[118,129],"experimental":[121],"designs":[122],"work,":[125],"together":[126],"conceptual":[131],"statistical":[133],"problems,":[134],"specific":[138],"guidelines":[139],"future":[141],"work.":[142],"(3)":[143],"methodology":[145,191],"criteria":[150],"on":[151],"equal":[153],"basis,":[154],"which":[155],"accounts":[156],"without":[161],"directly":[162],"manipulating":[163],"it":[164],"through":[165],"unrealistic":[166],"stratification.":[167],"(4)":[168],"empirical":[170],"evaluation":[171],"compares":[173],"effectiveness":[175],"coverage-based":[177],"testing,":[178,180],"mutation-based":[179],"random":[182],"testing.":[183],"Additionally,":[184],"proposes":[187],"probabilistic":[188],"coupling,":[189],"representativeness":[195],"goals":[201],"given":[204],"approximating":[208],"fault-detection":[210],"probability":[211],"adequate":[213],"sets.":[215]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":13},{"year":2021,"cited_by_count":8}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}