{"id":"https://openalex.org/W2083222184","doi":"https://doi.org/10.1109/tse.2015.2421011","title":"The Risks of Coverage-Directed Test Case Generation","display_name":"The Risks of Coverage-Directed Test Case Generation","publication_year":2015,"publication_date":"2015-04-08","ids":{"openalex":"https://openalex.org/W2083222184","doi":"https://doi.org/10.1109/tse.2015.2421011","mag":"2083222184"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2015.2421011","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2015.2421011","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082659908","display_name":"Gregory Gay","orcid":"https://orcid.org/0000-0001-6794-9585"},"institutions":[{"id":"https://openalex.org/I155781252","display_name":"University of South Carolina","ror":"https://ror.org/02b6qw903","country_code":"US","type":"education","lineage":["https://openalex.org/I155781252"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Gregory Gay","raw_affiliation_strings":["Department of Computer Science & Engineering, University of South Carolina"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering, University of South Carolina","institution_ids":["https://openalex.org/I155781252"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008490992","display_name":"Matt Staats","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matt Staats","raw_affiliation_strings":["Google, Inc"],"affiliations":[{"raw_affiliation_string":"Google, Inc","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014360702","display_name":"Michael W. Whalen","orcid":"https://orcid.org/0000-0003-3824-1435"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Whalen","raw_affiliation_strings":["Department of Computer Science and Engineering, University of Minnesota"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of Minnesota","institution_ids":["https://openalex.org/I130238516"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110505437","display_name":"Mats P. E. Heimdahl","orcid":null},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mats P. E. Heimdahl","raw_affiliation_strings":["Department of Computer Science and Engineering, University of Minnesota"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of Minnesota","institution_ids":["https://openalex.org/I130238516"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5082659908"],"corresponding_institution_ids":["https://openalex.org/I155781252"],"apc_list":null,"apc_paid":null,"fwci":20.0168,"has_fulltext":false,"cited_by_count":99,"citation_normalized_percentile":{"value":0.99490741,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"41","issue":"8","first_page":"803","last_page":"819"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7450418472290039},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.708752453327179},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.6561282873153687},{"id":"https://openalex.org/keywords/fault-coverage","display_name":"Fault coverage","score":0.6483446359634399},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.6054286360740662},{"id":"https://openalex.org/keywords/random-testing","display_name":"Random testing","score":0.5909002423286438},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5562887191772461},{"id":"https://openalex.org/keywords/automatic-test-pattern-generation","display_name":"Automatic test pattern generation","score":0.5543592572212219},{"id":"https://openalex.org/keywords/avionics","display_name":"Avionics","score":0.5116260647773743},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.46579277515411377},{"id":"https://openalex.org/keywords/counterexample","display_name":"Counterexample","score":0.435067355632782},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.42887312173843384},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3627639412879944},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.18862080574035645},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.16041597723960876},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.15861520171165466},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.12934914231300354},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09331348538398743}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7450418472290039},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.708752453327179},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.6561282873153687},{"id":"https://openalex.org/C126953365","wikidata":"https://www.wikidata.org/wiki/Q5438152","display_name":"Fault coverage","level":3,"score":0.6483446359634399},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.6054286360740662},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.5909002423286438},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5562887191772461},{"id":"https://openalex.org/C17626397","wikidata":"https://www.wikidata.org/wiki/Q837455","display_name":"Automatic test pattern generation","level":3,"score":0.5543592572212219},{"id":"https://openalex.org/C15792166","wikidata":"https://www.wikidata.org/wiki/Q221329","display_name":"Avionics","level":2,"score":0.5116260647773743},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.46579277515411377},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.435067355632782},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.42887312173843384},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3627639412879944},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.18862080574035645},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.16041597723960876},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15861520171165466},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12934914231300354},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09331348538398743},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C134146338","wikidata":"https://www.wikidata.org/wiki/Q1815901","display_name":"Electronic circuit","level":2,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tse.2015.2421011","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2015.2421011","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:conservancy.umn.edu:11299/217451","is_oa":false,"landing_page_url":"https://conservancy.umn.edu/bitstreams/811a0f1a-bbc2-44e1-8b78-2530aecd6286/download","pdf_url":null,"source":{"id":"https://openalex.org/S4306402521","display_name":"University of Minnesota Digital Conservancy (University of Minnesota)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I130238516","host_organization_name":"University of Minnesota","host_organization_lineage":["https://openalex.org/I130238516"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Report"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4071842969","display_name":null,"funder_award_id":"CNS-1035715","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6125804149","display_name":null,"funder_award_id":"FNR/P10/03","funder_id":"https://openalex.org/F4320321038","funder_display_name":"Fonds National de la Recherche Luxembourg"},{"id":"https://openalex.org/G7265297449","display_name":null,"funder_award_id":"NNA06CB21A","funder_id":"https://openalex.org/F4320306101","funder_display_name":"National Aeronautics and Space Administration"},{"id":"https://openalex.org/G7609230754","display_name":null,"funder_award_id":"CNS-0931931","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7630411954","display_name":null,"funder_award_id":"CCF-0916583","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306101","display_name":"National Aeronautics and Space Administration","ror":"https://ror.org/027ka1x80"},{"id":"https://openalex.org/F4320321038","display_name":"Fonds National de la Recherche Luxembourg","ror":"https://ror.org/039z13y21"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W32887977","https://openalex.org/W157156687","https://openalex.org/W1491168419","https://openalex.org/W1510251747","https://openalex.org/W1600639883","https://openalex.org/W1821489642","https://openalex.org/W1875492031","https://openalex.org/W1976727279","https://openalex.org/W1979345446","https://openalex.org/W1980818933","https://openalex.org/W1985378987","https://openalex.org/W1985759744","https://openalex.org/W2008356116","https://openalex.org/W2009489720","https://openalex.org/W2011325955","https://openalex.org/W2012854851","https://openalex.org/W2013711971","https://openalex.org/W2032782696","https://openalex.org/W2041474397","https://openalex.org/W2045002843","https://openalex.org/W2055647675","https://openalex.org/W2055975932","https://openalex.org/W2063068372","https://openalex.org/W2075699551","https://openalex.org/W2082700856","https://openalex.org/W2096449544","https://openalex.org/W2096662969","https://openalex.org/W2102742419","https://openalex.org/W2104993088","https://openalex.org/W2106072155","https://openalex.org/W2114772245","https://openalex.org/W2114869486","https://openalex.org/W2119805979","https://openalex.org/W2121750555","https://openalex.org/W2125539770","https://openalex.org/W2134691366","https://openalex.org/W2135841285","https://openalex.org/W2136043644","https://openalex.org/W2138612284","https://openalex.org/W2139459831","https://openalex.org/W2150288151","https://openalex.org/W2152664489","https://openalex.org/W2158921232","https://openalex.org/W2159181605","https://openalex.org/W2159901505","https://openalex.org/W2162200351","https://openalex.org/W2172253171","https://openalex.org/W2261317850","https://openalex.org/W2601354821","https://openalex.org/W3148345308","https://openalex.org/W4206455946","https://openalex.org/W4232038577","https://openalex.org/W4233529774","https://openalex.org/W4234542549","https://openalex.org/W4236200536","https://openalex.org/W4237492309","https://openalex.org/W4239794150","https://openalex.org/W4252684946","https://openalex.org/W4285719527","https://openalex.org/W6601390065","https://openalex.org/W6629890555","https://openalex.org/W6636034698","https://openalex.org/W6675591204","https://openalex.org/W6679978559","https://openalex.org/W6680087179","https://openalex.org/W6683269512","https://openalex.org/W6683428326","https://openalex.org/W6692735405","https://openalex.org/W7052854618"],"related_works":["https://openalex.org/W2091833418","https://openalex.org/W2913077774","https://openalex.org/W4256030018","https://openalex.org/W4295918990","https://openalex.org/W2147058777","https://openalex.org/W2183567146","https://openalex.org/W2096843010","https://openalex.org/W2003385991","https://openalex.org/W2024254950","https://openalex.org/W2907001447"],"abstract_inverted_index":{"A":[0],"number":[1],"of":[2,13,37,69,74,90,121,140,156,164,170,224,232,241,247,254],"structural":[3,27,54,171,234],"coverage":[4,28,55,97,113,131,172,235,280],"criteria":[5,29,98,132],"have":[6,182],"been":[7,80],"proposed":[8],"to":[9,46,51,60,94,191,197,207,216,260],"measure":[10],"the":[11,17,35,67,88,148,168,222,225,230,239,264,273,285],"adequacy":[12],"testing":[14],"efforts.":[15],"In":[16,83],"avionics":[18],"and":[19,103,221,272,289],"other":[20],"critical":[21,270],"systems":[22],"domains,":[23],"test":[24,40,49,91,101,119,154,179,188,203,226,242,252,267,282,286,294],"suites":[25,92,120,155,189,204,243,253],"satisfying":[26],"are":[30,64,109],"mandated":[31],"by":[32],"standards.":[33],"With":[34],"advent":[36],"powerful":[38],"automated":[39],"generation":[41,102,106,180,240,283],"tools,":[42],"it":[43],"is":[44,114],"tempting":[45],"simply":[47],"generate":[48],"inputs":[50],"satisfy":[52,95],"these":[53],"criteria.":[56],"However,":[57],"while":[58],"techniques":[59],"produce":[61],"coverage-providing":[62,193],"tests":[63,108],"well":[65],"established,":[66],"effectiveness":[68,89],"such":[70],"approaches":[71],"in":[72,263,266,269,278],"terms":[73],"fault":[75,141,165,248],"detection":[76,249],"ability":[77],"has":[78],"not":[79],"adequately":[81],"studied.":[82],"this":[84],"work,":[85],"we":[86],"evaluate":[87],"generated":[93,111,205],"four":[96],"through":[99],"counterexample-based":[100],"a":[104,137,174,177,183,192,213],"random":[105,118,153,187,251],"approach-where":[107],"randomly":[110],"until":[112],"achieved-contrasted":[115],"against":[116],"purely":[117],"equal":[122,157,255],"size.":[123,256],"Our":[124],"results":[125,146],"yield":[126],"three":[127],"key":[128],"conclusions.":[129],"First,":[130],"satisfaction":[133],"alone":[134],"can":[135,181],"be":[136],"poor":[138],"indication":[139],"finding":[142],"effectiveness,":[143],"with":[144,186],"inconsistent":[145],"between":[147],"seven":[149],"case":[150],"examples":[151],"(and":[152],"size":[158],"often":[159],"providing":[160],"similar-or":[161],"even":[162],"higher-levels":[163],"finding).":[166],"Second,":[167],"use":[169],"as":[173],"supplement-rather":[175],"than":[176,202,250],"target-for":[178],"positive":[184],"impact,":[185],"reduced":[190],"subset":[194],"detecting":[195],"up":[196],"13.5":[198],"percent":[199],"more":[200,276],"faults":[201],"specifically":[206],"achieve":[208],"coverage.":[209],"Finally,":[210],"Observable":[211],"MC/DC,":[212],"criterion":[214],"designed":[215],"account":[217],"for":[218,238,275],"program":[219],"structure":[220,291],"selection":[223],"oracle,":[227],"can-in":[228],"part-address":[229],"failings":[231],"traditional":[233],"criteria,":[236,281],"allowing":[237],"achieving":[244],"higher":[245],"levels":[246],"These":[257],"observations":[258],"point":[259],"risks":[261],"inherent":[262],"increase":[265],"automation":[268],"systems,":[271],"need":[274],"research":[277],"how":[279],"approaches,":[284],"oracle":[287],"used,":[288],"system":[290],"jointly":[292],"influence":[293],"effectiveness.":[295]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":15},{"year":2017,"cited_by_count":10},{"year":2016,"cited_by_count":9},{"year":2015,"cited_by_count":2}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}