{"id":"https://openalex.org/W4389337887","doi":"https://doi.org/10.1145/3635713","title":"Assessing Effectiveness of Test Suites: What Do We Know and What Should We Do?","display_name":"Assessing Effectiveness of Test Suites: What Do We Know and What Should We Do?","publication_year":2023,"publication_date":"2023-12-05","ids":{"openalex":"https://openalex.org/W4389337887","doi":"https://doi.org/10.1145/3635713"},"language":"en","primary_location":{"id":"doi:10.1145/3635713","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3635713","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3635713","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3635713","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100756151","display_name":"Peng Zhang","orcid":"https://orcid.org/0000-0003-2902-2249"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]},{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Zhang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China and Huawei Technologies Co., Ltd, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-2902-2249","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China and Huawei Technologies Co., Ltd, Hangzhou, China","institution_ids":["https://openalex.org/I2250955327","https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100322763","display_name":"Yang Wang","orcid":"https://orcid.org/0000-0002-6771-2139"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Wang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-6771-2139","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047883946","display_name":"Xutong Liu","orcid":"https://orcid.org/0000-0002-3831-5505"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xutong Liu","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-3831-5505","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088373482","display_name":"Zeyu Lu","orcid":"https://orcid.org/0009-0001-7370-6421"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeyu Lu","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0001-7370-6421","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037791183","display_name":"Yibiao Yang","orcid":"https://orcid.org/0000-0003-1153-2013"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yibiao Yang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-1153-2013","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100360608","display_name":"Yanhui Li","orcid":"https://orcid.org/0000-0003-2282-7175"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanhui Li","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-2282-7175","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100443798","display_name":"Lin Chen","orcid":"https://orcid.org/0000-0003-2352-2226"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Chen","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-2352-2226","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100379621","display_name":"Ziyuan Wang","orcid":"https://orcid.org/0000-0002-0494-5285"},"institutions":[{"id":"https://openalex.org/I41198531","display_name":"Nanjing University of Posts and Telecommunications","ror":"https://ror.org/043bpky34","country_code":"CN","type":"education","lineage":["https://openalex.org/I41198531"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyuan Wang","raw_affiliation_strings":["Nanjing University of Posts and Telecommunications, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-0494-5285","affiliations":[{"raw_affiliation_string":"Nanjing University of Posts and Telecommunications, Nanjing, China","institution_ids":["https://openalex.org/I41198531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048309076","display_name":"Chang\u2010ai Sun","orcid":"https://orcid.org/0000-0003-3696-6176"},"institutions":[{"id":"https://openalex.org/I92403157","display_name":"University of Science and Technology Beijing","ror":"https://ror.org/02egmk993","country_code":"CN","type":"education","lineage":["https://openalex.org/I92403157"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chang-Ai Sun","raw_affiliation_strings":["University of Science and Technology Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3696-6176","affiliations":[{"raw_affiliation_string":"University of Science and Technology Beijing, Beijing, China","institution_ids":["https://openalex.org/I92403157"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047060039","display_name":"Xiao Yu","orcid":"https://orcid.org/0000-0002-4473-3068"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Yu","raw_affiliation_strings":["Huawei Technologies Co., Ltd, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-4473-3068","affiliations":[{"raw_affiliation_string":"Huawei Technologies Co., Ltd, Hangzhou, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031391841","display_name":"Yuming Zhou","orcid":"https://orcid.org/0000-0002-4645-2526"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuming Zhou","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-4645-2526","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2331,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.85138026,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"33","issue":"4","first_page":"1","last_page":"32"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.787665605545044},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7394095659255981},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6956896781921387},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.6351364850997925},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.587729811668396},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5693410634994507},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.47625231742858887},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4522753357887268},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.44461682438850403},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.44338059425354004},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40729862451553345},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.39839503169059753},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.38098543882369995},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3680014908313751},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11847251653671265}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.787665605545044},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7394095659255981},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6956896781921387},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.6351364850997925},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.587729811668396},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5693410634994507},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.47625231742858887},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4522753357887268},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.44461682438850403},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.44338059425354004},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40729862451553345},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.39839503169059753},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.38098543882369995},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3680014908313751},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11847251653671265},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3635713","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3635713","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3635713","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3635713","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3635713","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3635713","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.41999998688697815}],"awards":[{"id":"https://openalex.org/G2345010575","display_name":null,"funder_award_id":"62172202","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4177809309","display_name":null,"funder_award_id":"62172205, 62072194, 62172202, 62272221, 62272037","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4362810476","display_name":"\u57fa\u4e8e\u8715\u53d8\u5173\u7cfb\u7684\u7f16\u8bd1\u5de5\u5177\u94fe\u7f3a\u9677\u68c0\u6d4b\u6280\u672f\u7814\u7a76","funder_award_id":"62072194","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4849440505","display_name":null,"funder_award_id":"BK20231402","funder_id":"https://openalex.org/F4320322769","funder_display_name":"Natural Science Foundation of Jiangsu Province"},{"id":"https://openalex.org/G5718290890","display_name":null,"funder_award_id":"62272221","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5997361112","display_name":null,"funder_award_id":"62172205","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7744500962","display_name":null,"funder_award_id":"62272037","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321605","display_name":"Government of Jiangsu Province","ror":"https://ror.org/004svx814"},{"id":"https://openalex.org/F4320322769","display_name":"Natural Science Foundation of Jiangsu Province","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389337887.pdf","grobid_xml":"https://content.openalex.org/works/W4389337887.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W1579202050","https://openalex.org/W1984481956","https://openalex.org/W2026097184","https://openalex.org/W2028990673","https://openalex.org/W2049695835","https://openalex.org/W2063068372","https://openalex.org/W2075592454","https://openalex.org/W2106072155","https://openalex.org/W2108557864","https://openalex.org/W2114733864","https://openalex.org/W2121084350","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2143002644","https://openalex.org/W2168561184","https://openalex.org/W2340019743","https://openalex.org/W2346933435","https://openalex.org/W2546730260","https://openalex.org/W2546792321","https://openalex.org/W2598621163","https://openalex.org/W2604348044","https://openalex.org/W2739864926","https://openalex.org/W2788962378","https://openalex.org/W2920244696","https://openalex.org/W2949400639","https://openalex.org/W2955667094","https://openalex.org/W3045296135","https://openalex.org/W3111827872","https://openalex.org/W3127336347","https://openalex.org/W4239327426","https://openalex.org/W4240996491","https://openalex.org/W4242868108","https://openalex.org/W4243177274","https://openalex.org/W4247035379"],"related_works":["https://openalex.org/W1485837041","https://openalex.org/W2151687600","https://openalex.org/W3204955359","https://openalex.org/W2900347730","https://openalex.org/W1598160211","https://openalex.org/W2018145554","https://openalex.org/W1524028411","https://openalex.org/W2138059712","https://openalex.org/W2127248783","https://openalex.org/W2067499928"],"abstract_inverted_index":{"Background.":[0],"Software":[1],"testing":[2],"is":[3,36,109,186],"a":[4,27,43,51,140,165,208],"critical":[5],"activity":[6],"for":[7,112,130,163,195,233,340],"ensuring":[8],"the":[9,18,40,95,98,123,160,192,197,225,229,234,241,245,260,291,301,332,348],"quality":[10],"and":[11,103,181,214,281,296,342],"reliability":[12],"of":[13,20,29,42,73,75,97,126,172,210,262,314],"software":[14],"systems.":[15],"To":[16,134],"evaluate":[17,272],"effectiveness":[19,168,201,232,266,276,318,345],"different":[21,65,263],"test":[22,128,166,179,203,212,236,264,274,306,317,344],"suites,":[23,180],"researchers":[24,53,91],"have":[25,80],"developed":[26],"variety":[28],"metrics.":[30,251,267,284],"Problem.":[31],"However,":[32],"comparing":[33,71,127,343],"these":[34],"metrics":[35,303,346],"challenging":[37],"due":[38],"to":[39,64,87,93,158,227,258,271,304],"lack":[41],"standardized":[44,333],"evaluation":[45,334],"framework":[46,141,335],"including":[47,278],"comprehensive":[48],"factors.":[49],"As":[50],"result,":[52],"often":[54],"focus":[55],"on":[56,290],"single":[57],"factors":[58],"(e.g.,":[59],"size),":[60],"which":[61],"finally":[62],"leads":[63],"or":[66],"even":[67],"contradictory":[68],"conclusions.":[69],"After":[70],"dozens":[72],"pieces":[74],"work":[76],"in":[77,200,220,231,326,347],"detail,":[78],"we":[79,138,255],"found":[81],"two":[82,246,250],"main":[83],"problems":[84],"most":[85],"troubling":[86],"our":[88,131],"community:":[89],"(1)":[90],"tend":[92],"oversimplify":[94],"description":[96],"ground":[99,176,193,217],"truth":[100,194,218],"they":[101],"use,":[102],"(2)":[104],"data":[105],"involving":[106],"real":[107,198,292,315],"defects":[108],"not":[110],"suitable":[111],"analysis":[113],"using":[114,311],"traditional":[115],"statistical":[116],"indicators.":[117],"Objective.":[118],"We":[119,268,329],"aim":[120],"at":[121],"scrutinizing":[122],"whole":[124],"process":[125],"suites":[129,213],"community.":[132],"Method.":[133],"hit":[135],"this":[136],"aim,":[137],"propose":[139],"ASSENT":[142,170,270,336],"(ev":[143],"A":[144],"luating":[145],"te":[146],"S":[147,149],"t":[148],"uite":[150],"E":[151],"ffective":[152],"N":[153],"ess":[154],"me":[155],"T":[156],"rics)":[157],"guide":[159],"follow-up":[161],"research":[162],"evaluating":[164,341],"suite":[167,265,275,307],"metric.":[169],"consists":[171],"three":[173],"fundamental":[174],"components:":[175],"truth,":[177],"benchmark":[178,211],"agreement":[182,242],"indicator.":[183],"Its":[184],"functioning":[185],"as":[187],"follows:":[188],"first,":[189],"users":[190,206,223,239],"clarify":[191],"determining":[196],"order":[199,219,230],"among":[202],"suites.":[204,237],"Second,":[205],"generate":[207],"set":[209],"derive":[215,228],"their":[216],"effectiveness.":[221,308],"Third,":[222],"use":[224],"metric":[226],"same":[235],"Finally,":[238],"calculate":[240],"indicator":[243],"between":[244],"orders":[247],"derived":[248],"by":[249,310,322],"Result.":[252],"With":[253],"ASSENT,":[254],"are":[256,300],"able":[257],"compare":[259],"accuracy":[261],"apply":[269],"representative":[273],"metrics,":[277],"mutation":[279,294,298],"score":[280,299],"code":[282],"coverage":[283],"Our":[285],"results":[286],"show":[287],"that,":[288],"based":[289],"faults,":[293,316],"score,":[295],"subsuming":[297],"best":[302],"quantify":[305],"Meanwhile,":[309],"mutants":[312],"instead":[313],"will":[319],"be":[320,338],"overestimated":[321],"more":[323],"than":[324],"20%":[325],"values.":[327],"Conclusion.":[328],"recommend":[330],"that":[331],"should":[337],"used":[339],"future":[349],"work.":[350]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}