{"id":"https://openalex.org/W1984481956","doi":"https://doi.org/10.1145/2660767","title":"Guidelines for Coverage-Based Comparisons of Non-Adequate Test Suites","display_name":"Guidelines for Coverage-Based Comparisons of Non-Adequate Test Suites","publication_year":2015,"publication_date":"2015-09-02","ids":{"openalex":"https://openalex.org/W1984481956","doi":"https://doi.org/10.1145/2660767","mag":"1984481956"},"language":"en","primary_location":{"id":"doi:10.1145/2660767","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2660767","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063052820","display_name":"Milos Gligoric","orcid":"https://orcid.org/0000-0002-5894-7649"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Milos Gligoric","raw_affiliation_strings":["University of Illinois at Urbana-Champaign","University of Illinois at Urbana Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"University of Illinois at Urbana Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026932942","display_name":"Alex Groce","orcid":"https://orcid.org/0000-0003-0273-4668"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Groce","raw_affiliation_strings":["Oregon State University","Oregon State University>"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"Oregon State University>","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026522734","display_name":"Chaoqiang Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chaoqiang Zhang","raw_affiliation_strings":["Oregon State University","Oregon State University>"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"Oregon State University>","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038725531","display_name":"Rohan Sharma","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rohan Sharma","raw_affiliation_strings":["University of Illinois at Urbana-Champaign","University of Illinois at Urbana Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"University of Illinois at Urbana Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090034689","display_name":"Mohammad Amin Alipour","orcid":"https://orcid.org/0000-0002-6479-7202"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammad Amin Alipour","raw_affiliation_strings":["Oregon State University","Oregon State University>"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"Oregon State University>","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081289583","display_name":"Darko Marinov","orcid":"https://orcid.org/0000-0001-5023-3492"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Darko Marinov","raw_affiliation_strings":["University of Illinois at Urbana-Champaign","University of Illinois at Urbana Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"University of Illinois at Urbana Champaign","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.5679,"has_fulltext":false,"cited_by_count":65,"citation_normalized_percentile":{"value":0.97623282,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"24","issue":"4","first_page":"1","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7447823286056519},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.7114277482032776},{"id":"https://openalex.org/keywords/equivalence","display_name":"Equivalence (formal languages)","score":0.5154638886451721},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.49181273579597473},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.48710131645202637},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4285210967063904},{"id":"https://openalex.org/keywords/predicate","display_name":"Predicate (mathematical logic)","score":0.4206344485282898},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.33819878101348877},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.19648075103759766},{"id":"https://openalex.org/keywords/discrete-mathematics","display_name":"Discrete mathematics","score":0.14130693674087524},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.0681990385055542}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7447823286056519},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.7114277482032776},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.5154638886451721},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.49181273579597473},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.48710131645202637},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4285210967063904},{"id":"https://openalex.org/C140146324","wikidata":"https://www.wikidata.org/wiki/Q1144319","display_name":"Predicate (mathematical logic)","level":2,"score":0.4206344485282898},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.33819878101348877},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19648075103759766},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.14130693674087524},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.0681990385055542},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2660767","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2660767","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W157798378","https://openalex.org/W198131454","https://openalex.org/W1158844662","https://openalex.org/W1486172410","https://openalex.org/W1500987966","https://openalex.org/W1507039213","https://openalex.org/W1560412976","https://openalex.org/W1593025666","https://openalex.org/W1841818470","https://openalex.org/W1857528250","https://openalex.org/W1902482618","https://openalex.org/W1971137495","https://openalex.org/W1976727279","https://openalex.org/W1985514943","https://openalex.org/W1988537734","https://openalex.org/W1995804606","https://openalex.org/W1995910633","https://openalex.org/W2006027584","https://openalex.org/W2008228118","https://openalex.org/W2012997183","https://openalex.org/W2013711971","https://openalex.org/W2019368525","https://openalex.org/W2028536280","https://openalex.org/W2049082552","https://openalex.org/W2049695835","https://openalex.org/W2054751837","https://openalex.org/W2063068372","https://openalex.org/W2072596778","https://openalex.org/W2078483536","https://openalex.org/W2082653518","https://openalex.org/W2086000360","https://openalex.org/W2089139117","https://openalex.org/W2095710561","https://openalex.org/W2099018580","https://openalex.org/W2101134669","https://openalex.org/W2102742419","https://openalex.org/W2105691657","https://openalex.org/W2105832118","https://openalex.org/W2106072155","https://openalex.org/W2107089133","https://openalex.org/W2107709519","https://openalex.org/W2108112890","https://openalex.org/W2108557864","https://openalex.org/W2110068396","https://openalex.org/W2114208144","https://openalex.org/W2119228825","https://openalex.org/W2120713211","https://openalex.org/W2124300773","https://openalex.org/W2125727889","https://openalex.org/W2132627437","https://openalex.org/W2133074421","https://openalex.org/W2133612077","https://openalex.org/W2134691366","https://openalex.org/W2135841285","https://openalex.org/W2138059712","https://openalex.org/W2138428785","https://openalex.org/W2139763472","https://openalex.org/W2142679626","https://openalex.org/W2144939470","https://openalex.org/W2145922831","https://openalex.org/W2151704521","https://openalex.org/W2151967815","https://openalex.org/W2154650451","https://openalex.org/W2156127351","https://openalex.org/W2156305884","https://openalex.org/W2168561184","https://openalex.org/W2169565104","https://openalex.org/W2170204382","https://openalex.org/W2295658119","https://openalex.org/W2295903414","https://openalex.org/W2327744707","https://openalex.org/W2472622445","https://openalex.org/W2912296587","https://openalex.org/W3145128584","https://openalex.org/W4239181501","https://openalex.org/W4246385123","https://openalex.org/W4247035379","https://openalex.org/W4285719527","https://openalex.org/W4298375254","https://openalex.org/W4376850198"],"related_works":["https://openalex.org/W1598160211","https://openalex.org/W2067499928","https://openalex.org/W1524028411","https://openalex.org/W2138059712","https://openalex.org/W2018145554","https://openalex.org/W2127248783","https://openalex.org/W4256285596","https://openalex.org/W1793348505","https://openalex.org/W3047487993","https://openalex.org/W2366485096"],"abstract_inverted_index":{"A":[0,36,55],"fundamental":[1],"question":[2],"in":[3,101,238],"software":[4],"testing":[5,302],"research":[6,322],"is":[7,64,108,141,186],"how":[8,48],"to":[9,143,145,189,305],"compare":[10,28,146,190],"test":[11,24,29,44,79,130,147,274,307],"suites,":[12],"often":[13],"as":[14,227,232,234,312,314],"a":[15,41,51,217,260],"means":[16],"for":[17,124,171,183,301,315,321],"comparing":[18],"test-generation":[19],"techniques":[20],"that":[21,57,120],"produce":[22],"those":[23],"suites.":[25,275],"Researchers":[26],"frequently":[27],"suites":[30,91,107,151,191,308],"by":[31],"measuring":[32],"their":[33],"coverage":[34,37,72,122,161,291,310,319],".":[35,67],"criterion":[38],"C":[39,84,86,89,97,135,137,172,184,265],"provides":[40],"set":[42,219,261],"of":[43,60,71,128,220,249,262],"requirements":[45,50,63],"and":[46,85,136,173,229,252,264,271,292],"measures":[47],"many":[49,102],"given":[52,81,132],"suite":[53,56],"satisfies.":[54],"satisfies":[58],"100%":[59],"the":[61,116,125],"feasible":[62],"called":[65],"C-adequate":[66],"Previous":[68],"rigorous":[69],"evaluations":[70],"criteria":[73,83,123,134,225,236,242,256,287,311,320],"mostly":[74],"focused":[75],"on":[76,92,193,204,244,259,283],"such":[77,226],"adequate":[78,106],"suites:":[80,131],"two":[82,133,286],"\u2032,":[87,138,176,185,207],"are":[88,257],"-adequate":[90],"average":[93],"more":[94],"effective":[95],"than":[96],"\u2032-adequate":[98],"suites?":[99,148],"However,":[100],"realistic":[103],"cases,":[104],"producing":[105],"impractical":[109],"or":[110,202],"even":[111],"impossible.":[112],"This":[113],"article":[114],"presents":[115],"first":[117],"extensive":[118],"study":[119],"evaluates":[121],"common":[126],"case":[127],"non-adequate":[129],"which":[139],"one":[140],"better":[142,188],"use":[144],"Namely,":[149],"if":[150],"T":[152,155,158],"1":[153,164,175,195,206],",":[154,165,196],"2":[156,167,178,198,209],",\u2026,":[157,168,199],"n":[159,170,181,201,212],"have":[160],"values":[162],"c":[163,166,169,174,177,180,194,197,200,205,208,211],"\u2032,\u2026,":[179,210],"\u2032":[182,213],"it":[187],"based":[192,203,243],"?":[214],"We":[215,298],"evaluate":[216,306],"large":[218],"plausible":[221],"criteria,":[222],"including":[223,241],"basic":[224],"statement":[228],"branch":[230,290],"coverage,":[231],"well":[233,313],"stronger":[235],"used":[237],"recent":[239],"studies,":[240],"program":[245],"paths,":[246],"equivalence":[247],"classes":[248],"covered":[250],"statements,":[251],"predicate":[253],"states.":[254],"The":[255,276],"evaluated":[258],"Java":[263],"programs":[266],"with":[267],"both":[268],"manually":[269],"written":[270],"automatically":[272],"generated":[273],"evaluation":[277],"uses":[278],"three":[279],"correlation":[280],"measures.":[281],"Based":[282],"these":[284],"experiments,":[285],"perform":[288],"best:":[289],"an":[293],"intraprocedural":[294],"acyclic":[295],"path":[296],"coverage.":[297],"provide":[299],"guidelines":[300],"researchers":[303,317],"aiming":[304],"using":[309],"other":[316],"evaluating":[318],"use.":[323]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":6},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}