{"id":"https://openalex.org/W4389162054","doi":"https://doi.org/10.1145/3611643.3613089","title":"Keeping Mutation Test Suites Consistent and Relevant with Long-Standing Mutants","display_name":"Keeping Mutation Test Suites Consistent and Relevant with Long-Standing Mutants","publication_year":2023,"publication_date":"2023-11-30","ids":{"openalex":"https://openalex.org/W4389162054","doi":"https://doi.org/10.1145/3611643.3613089"},"language":"en","primary_location":{"id":"doi:10.1145/3611643.3613089","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3611643.3613089","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3611643.3613089","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3611643.3613089","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015192836","display_name":"Milo\u0161 Ojdani\u0107","orcid":"https://orcid.org/0000-0001-8213-3381"},"institutions":[{"id":"https://openalex.org/I186903577","display_name":"University of Luxembourg","ror":"https://ror.org/036x5ad56","country_code":"LU","type":"education","lineage":["https://openalex.org/I186903577"]}],"countries":["LU"],"is_corresponding":true,"raw_author_name":"Milos Ojdanic","raw_affiliation_strings":["University of Luxembourg, Luxembourg City, Luxembourg"],"raw_orcid":"https://orcid.org/0000-0001-8213-3381","affiliations":[{"raw_affiliation_string":"University of Luxembourg, Luxembourg City, Luxembourg","institution_ids":["https://openalex.org/I186903577"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081145634","display_name":"Mike Papadakis","orcid":"https://orcid.org/0000-0003-1852-2547"},"institutions":[{"id":"https://openalex.org/I186903577","display_name":"University of Luxembourg","ror":"https://ror.org/036x5ad56","country_code":"LU","type":"education","lineage":["https://openalex.org/I186903577"]}],"countries":["LU"],"is_corresponding":false,"raw_author_name":"Mike Papadakis","raw_affiliation_strings":["University of Luxembourg, Luxembourg City, Luxembourg"],"raw_orcid":"https://orcid.org/0000-0003-1852-2547","affiliations":[{"raw_affiliation_string":"University of Luxembourg, Luxembourg City, Luxembourg","institution_ids":["https://openalex.org/I186903577"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000019783","display_name":"Mark Harman","orcid":"https://orcid.org/0000-0002-5864-4488"},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Harman","raw_affiliation_strings":["Meta Platforms, London, UK / University College London, London, UK"],"raw_orcid":"https://orcid.org/0000-0002-5864-4488","affiliations":[{"raw_affiliation_string":"Meta Platforms, London, UK / University College London, London, UK","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5015192836"],"corresponding_institution_ids":["https://openalex.org/I186903577"],"apc_list":null,"apc_paid":null,"fwci":0.2125,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.62407661,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"2067","last_page":"2071"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11005","display_name":"Radiation Effects in Electronics","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mutant","display_name":"Mutant","score":0.8334514498710632},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.784221887588501},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6968257427215576},{"id":"https://openalex.org/keywords/mutation-testing","display_name":"Mutation testing","score":0.676384449005127},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.6458258628845215},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6275391578674316},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6021445393562317},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.5022754669189453},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.49374088644981384},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.44951653480529785},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.36222970485687256},{"id":"https://openalex.org/keywords/computational-biology","display_name":"Computational biology","score":0.343273401260376},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.24751660227775574},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24167075753211975},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2351571023464203},{"id":"https://openalex.org/keywords/genetics","display_name":"Genetics","score":0.23301759362220764},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.2110159695148468},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.16620874404907227}],"concepts":[{"id":"https://openalex.org/C143065580","wikidata":"https://www.wikidata.org/wiki/Q3285695","display_name":"Mutant","level":3,"score":0.8334514498710632},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.784221887588501},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6968257427215576},{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.676384449005127},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.6458258628845215},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6275391578674316},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6021445393562317},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5022754669189453},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.49374088644981384},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.44951653480529785},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.36222970485687256},{"id":"https://openalex.org/C70721500","wikidata":"https://www.wikidata.org/wiki/Q177005","display_name":"Computational biology","level":1,"score":0.343273401260376},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.24751660227775574},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24167075753211975},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2351571023464203},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.23301759362220764},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.2110159695148468},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.16620874404907227},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3611643.3613089","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3611643.3613089","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3611643.3613089","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:orbilu.uni.lu:10993/58647","is_oa":true,"landing_page_url":"https://orbilu.uni.lu/handle/10993/58647","pdf_url":"https://orbilu.uni.lu/bitstream/10993/58647/1/article.pdf","source":{"id":"https://openalex.org/S4306401815","display_name":"Open Repository and Bibliography (University of Luxembourg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I186903577","host_organization_name":"University of Luxembourg","host_organization_lineage":["https://openalex.org/I186903577"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Keeping Mutation Test Suites Consistent and Relevant with Long-Standing Mutants (2023-11-30); The ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE 2023), San Francisco, United States - California [US-CA], Sun 3 - Sat 9 December 2023","raw_type":"peer reviewed"}],"best_oa_location":{"id":"doi:10.1145/3611643.3613089","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3611643.3613089","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3611643.3613089","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389162054.pdf","grobid_xml":"https://content.openalex.org/works/W4389162054.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2001936191","https://openalex.org/W2080097084","https://openalex.org/W2135841285","https://openalex.org/W2148646465","https://openalex.org/W2160431913","https://openalex.org/W2166541753","https://openalex.org/W2170260129","https://openalex.org/W2466388701","https://openalex.org/W2502012205","https://openalex.org/W2608579592","https://openalex.org/W2616524285","https://openalex.org/W2725449579","https://openalex.org/W2787216049","https://openalex.org/W2787694176","https://openalex.org/W2793776506","https://openalex.org/W2953998535","https://openalex.org/W2960013457","https://openalex.org/W2962975379","https://openalex.org/W2998011150","https://openalex.org/W3102559360","https://openalex.org/W3161290919","https://openalex.org/W4206242116","https://openalex.org/W4229850662","https://openalex.org/W4238681179","https://openalex.org/W4280606181","https://openalex.org/W4281924103"],"related_works":["https://openalex.org/W1485837041","https://openalex.org/W3204955359","https://openalex.org/W2038379963","https://openalex.org/W4251258648","https://openalex.org/W1995023328","https://openalex.org/W2731612402","https://openalex.org/W2960013457","https://openalex.org/W1598160211","https://openalex.org/W1490208943","https://openalex.org/W2599144236"],"abstract_inverted_index":{"Mutation":[0],"testing":[1],"has":[2],"been":[3],"demonstrated":[4],"to":[5,25,28,52,99],"be":[6,26,46,116],"one":[7],"of":[8,141,156],"the":[9,15,134,139,152],"most":[10],"powerful":[11],"fault-revealing":[12],"tools":[13],"in":[14,75,80,122],"tester's":[16],"tool":[17],"kit.":[18],"Much":[19],"previous":[20],"work":[21],"implicitly":[22],"assumed":[23],"it":[24,50,98],"sufficient":[27],"re-compute":[29],"mutant":[30,40,63,93,113,123,142],"suites":[31,114,143],"per":[32],"release.":[33],"Sadly,":[34],"this":[35,73],"makes":[36],"mutation":[37,105,157],"results":[38,131],"inconsistent;":[39],"scores":[41],"from":[42],"each":[43],"release":[44],"cannot":[45],"directly":[47],"compared,":[48],"making":[49],"harder":[51],"measure":[53,72,95],"test":[54,128],"improvement.":[55],"Furthermore,":[56],"regular":[57],"code":[58],"change":[59],"means":[60],"that":[61,85,133],"a":[62,92,119],"suite's":[64],"relevance":[65,76,124,155],"will":[66],"naturally":[67],"degrade":[68],"over":[69,125],"time.":[70],"We":[71,90,107],"degradation":[74],"for":[77],"143,500":[78],"mutants":[79],"4":[81],"non-trivial":[82],"systems,":[83],"finding":[84],"52%":[86],"degrade,":[87],"on":[88,147],"average.":[89],"introduce":[91],"brittleness":[94],"and":[96,103,144,154],"use":[97],"audit":[100],"software":[101],"systems":[102],"their":[104],"suites.":[106],"also":[108],"demonstrate":[109],"how":[110],"consistent-by-construction":[111],"long-standing":[112,148],"can":[115],"identified":[117],"with":[118],"10x":[120],"improvement":[121],"an":[126],"arbitrary":[127],"suite.":[129],"Our":[130],"indicate":[132],"research":[135],"community":[136],"should":[137],"avoid":[138],"re-computation":[140],"focus,":[145],"instead,":[146],"mutants,":[149],"thereby":[150],"improving":[151],"consistency":[153],"testing.":[158]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}