{"id":"https://openalex.org/W2960013457","doi":"https://doi.org/10.1145/3293882.3330568","title":"Mitigating the effects of flaky tests on mutation testing","display_name":"Mitigating the effects of flaky tests on mutation testing","publication_year":2019,"publication_date":"2019-07-10","ids":{"openalex":"https://openalex.org/W2960013457","doi":"https://doi.org/10.1145/3293882.3330568","mag":"2960013457"},"language":"en","primary_location":{"id":"doi:10.1145/3293882.3330568","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3293882.3330568","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM SIGSOFT International Symposium on Software Testing and Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081483504","display_name":"August Shi","orcid":"https://orcid.org/0000-0001-8239-3124"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"August Shi","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053080959","display_name":"Jonathan Bell","orcid":"https://orcid.org/0000-0002-1187-9298"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jonathan Bell","raw_affiliation_strings":["George Mason University, USA"],"affiliations":[{"raw_affiliation_string":"George Mason University, USA","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081289583","display_name":"Darko Marinov","orcid":"https://orcid.org/0000-0001-5023-3492"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Darko Marinov","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5081483504"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":10.0267,"has_fulltext":false,"cited_by_count":54,"citation_normalized_percentile":{"value":0.98463481,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"112","last_page":"122"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.8860845565795898},{"id":"https://openalex.org/keywords/mutation-testing","display_name":"Mutation testing","score":0.7407112121582031},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.6749974489212036},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6305598020553589},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5990967154502869},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.555610179901123},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.49199625849723816},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.49087196588516235},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.4526301920413971},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.4472917318344116},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4438636004924774},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.43447747826576233},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.432604044675827},{"id":"https://openalex.org/keywords/mutant","display_name":"Mutant","score":0.41064319014549255},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3984057605266571},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.22080102562904358},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.21265915036201477},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.17193669080734253},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.16689446568489075},{"id":"https://openalex.org/keywords/genetics","display_name":"Genetics","score":0.15723159909248352},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.14817476272583008}],"concepts":[{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.8860845565795898},{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.7407112121582031},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.6749974489212036},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6305598020553589},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5990967154502869},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.555610179901123},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.49199625849723816},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.49087196588516235},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.4526301920413971},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.4472917318344116},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4438636004924774},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.43447747826576233},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.432604044675827},{"id":"https://openalex.org/C143065580","wikidata":"https://www.wikidata.org/wiki/Q3285695","display_name":"Mutant","level":3,"score":0.41064319014549255},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3984057605266571},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.22080102562904358},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.21265915036201477},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.17193669080734253},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.16689446568489075},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.15723159909248352},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.14817476272583008},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3293882.3330568","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3293882.3330568","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM SIGSOFT International Symposium on Software Testing and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1002643292","display_name":null,"funder_award_id":"CCF1421503, CNS-1646305, CNS-1740916, CCF-1763788, CCF-1763822, OAC-1839010","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1976954202","https://openalex.org/W1995804606","https://openalex.org/W2002644983","https://openalex.org/W2004222258","https://openalex.org/W2008479535","https://openalex.org/W2014011458","https://openalex.org/W2026926213","https://openalex.org/W2040923543","https://openalex.org/W2044527526","https://openalex.org/W2050127001","https://openalex.org/W2053873374","https://openalex.org/W2054751837","https://openalex.org/W2059018540","https://openalex.org/W2061781275","https://openalex.org/W2073655054","https://openalex.org/W2077258409","https://openalex.org/W2079922024","https://openalex.org/W2091556188","https://openalex.org/W2091594042","https://openalex.org/W2129377409","https://openalex.org/W2129761255","https://openalex.org/W2135841285","https://openalex.org/W2141852905","https://openalex.org/W2142557127","https://openalex.org/W2156723666","https://openalex.org/W2164105290","https://openalex.org/W2171451409","https://openalex.org/W2240451833","https://openalex.org/W2352511489","https://openalex.org/W2466388701","https://openalex.org/W2477215876","https://openalex.org/W2537787699","https://openalex.org/W2546867968","https://openalex.org/W2607160846","https://openalex.org/W2767787791","https://openalex.org/W2786619380","https://openalex.org/W2786944871","https://openalex.org/W2794756405","https://openalex.org/W2809559400","https://openalex.org/W2878316859","https://openalex.org/W2884023932","https://openalex.org/W2888078871","https://openalex.org/W2901387825","https://openalex.org/W2951170114","https://openalex.org/W3101548319","https://openalex.org/W4238681179","https://openalex.org/W4252613971","https://openalex.org/W4394331319"],"related_works":["https://openalex.org/W4295918990","https://openalex.org/W2137109811","https://openalex.org/W4313447549","https://openalex.org/W2028796071","https://openalex.org/W2410893741","https://openalex.org/W2067499928","https://openalex.org/W1888619389","https://openalex.org/W1968494916","https://openalex.org/W4376482197","https://openalex.org/W2794522096"],"abstract_inverted_index":{"Mutation":[0,17,50],"testing":[1,18,51,106,154,171],"is":[2],"widely":[3],"used":[4],"in":[5,59,81,109,113],"research":[6],"as":[7],"a":[8,33,37,68,72],"metric":[9],"for":[10],"evaluating":[11],"the":[12,20,28,42,48,65,82,96,152,168,184],"quality":[13],"of":[14,27,41,61,67,84,130,186],"test":[15,21,34,69,103],"suites.":[16],"runs":[19],"suite":[22,35],"on":[23,47,95,123,158,176],"generated":[24],"mutants":[25,189],"(variants":[26],"code":[29,98],"under":[30,99],"test),":[31],"where":[32],"kills":[36],"mutant":[38],"if":[39],"any":[40],"tests":[43,55],"fail":[44],"when":[45,93],"run":[46,94],"mutant.":[49,74],"implicitly":[52],"assumes":[53],"that":[54,148,180],"exhibit":[56],"deterministic":[57],"behavior,":[58],"terms":[60],"their":[62],"coverage":[63],"and":[64,128],"outcome":[66],"(not)":[70],"killing":[71],"certain":[73],"Such":[75],"an":[76,134],"assumption":[77],"does":[78],"not":[79],"hold":[80],"presence":[83],"flaky":[85,143],"tests,":[86],"whose":[87],"outcomes":[88],"can":[89,107],"non-deterministically":[90],"differ":[91],"even":[92],"same":[97],"test.":[100],"Without":[101],"reliable":[102],"outcomes,":[104],"mutation":[105,116,153,170],"result":[108],"unreliable":[110],"results,":[111],"e.g.,":[112],"our":[114,164,181],"experiments,":[115],"scores":[117],"vary":[118],"by":[119,166,190],"four":[120],"percentage":[121],"points":[122],"average":[124],"between":[125],"repeated":[126],"executions,":[127],"9%":[129],"mutant-test":[131],"pairs":[132],"have":[133],"unknown":[135],"status.":[136],"Many":[137],"modern":[138],"software":[139],"projects":[140,178],"suffer":[141],"from":[142],"tests.":[144,161],"We":[145,162],"propose":[146],"techniques":[147,165,182],"manage":[149],"flakiness":[150],"throughout":[151],"process,":[155],"largely":[156],"based":[157],"strategically":[159],"re-running":[160],"implement":[163],"modifying":[167],"open-source":[169],"tool,":[172],"PIT.":[173],"Our":[174],"evaluation":[175],"30":[177],"shows":[179],"reduce":[183],"number":[185],"\"unknown\"":[187],"(flaky)":[188],"79.4%.":[191]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":7},{"year":2019,"cited_by_count":2}],"updated_date":"2026-03-30T08:08:38.191290","created_date":"2025-10-10T00:00:00"}