{"id":"https://openalex.org/W2026926213","doi":"https://doi.org/10.1145/2635868.2635920","title":"An empirical analysis of flaky tests","display_name":"An empirical analysis of flaky tests","publication_year":2014,"publication_date":"2014-11-04","ids":{"openalex":"https://openalex.org/W2026926213","doi":"https://doi.org/10.1145/2635868.2635920","mag":"2026926213"},"language":"en","primary_location":{"id":"doi:10.1145/2635868.2635920","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2635868.2635920","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027636949","display_name":"Qingzhou Luo","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Qingzhou Luo","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028388085","display_name":"Farah Hariri","orcid":"https://orcid.org/0000-0001-8486-7151"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Farah Hariri","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011545984","display_name":"Lamyaa Eloussi","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lamyaa Eloussi","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081289583","display_name":"Darko Marinov","orcid":"https://orcid.org/0000-0001-5023-3492"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Darko Marinov","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5027636949"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":30.269,"has_fulltext":false,"cited_by_count":406,"citation_normalized_percentile":{"value":0.9955196,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"643","last_page":"653"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/regression-testing","display_name":"Regression testing","score":0.8916754722595215},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6109579205513},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5800830721855164},{"id":"https://openalex.org/keywords/risk-based-testing","display_name":"Risk-based testing","score":0.509019136428833},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4761766791343689},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.46024221181869507},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"Regression analysis","score":0.4498976767063141},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.4174036681652069},{"id":"https://openalex.org/keywords/root-cause","display_name":"Root cause","score":0.4144745469093323},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.39835524559020996},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3659254312515259},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3272291421890259},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.22816067934036255},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.22465142607688904},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.17335647344589233},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.15232717990875244},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.1219450831413269},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08773809671401978}],"concepts":[{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.8916754722595215},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6109579205513},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5800830721855164},{"id":"https://openalex.org/C37945671","wikidata":"https://www.wikidata.org/wiki/Q7336207","display_name":"Risk-based testing","level":5,"score":0.509019136428833},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4761766791343689},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.46024221181869507},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.4498976767063141},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4174036681652069},{"id":"https://openalex.org/C84945661","wikidata":"https://www.wikidata.org/wiki/Q7366567","display_name":"Root cause","level":2,"score":0.4144745469093323},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.39835524559020996},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3659254312515259},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3272291421890259},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.22816067934036255},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.22465142607688904},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.17335647344589233},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.15232717990875244},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.1219450831413269},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08773809671401978},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2635868.2635920","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2635868.2635920","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W146863617","https://openalex.org/W1966151649","https://openalex.org/W1994816063","https://openalex.org/W1999808610","https://openalex.org/W2002934700","https://openalex.org/W2028513599","https://openalex.org/W2029055005","https://openalex.org/W2050127001","https://openalex.org/W2067377566","https://openalex.org/W2076879269","https://openalex.org/W2079811826","https://openalex.org/W2100627043","https://openalex.org/W2101161997","https://openalex.org/W2107500604","https://openalex.org/W2110229593","https://openalex.org/W2118437239","https://openalex.org/W2129164226","https://openalex.org/W2131112324","https://openalex.org/W2141852905","https://openalex.org/W2149433427","https://openalex.org/W2156633971","https://openalex.org/W2158082837","https://openalex.org/W2159765571","https://openalex.org/W4210712189"],"related_works":["https://openalex.org/W4367553652","https://openalex.org/W2362944210","https://openalex.org/W2625990769","https://openalex.org/W4232229155","https://openalex.org/W4210852310","https://openalex.org/W2131290059","https://openalex.org/W2162694379","https://openalex.org/W2359673812","https://openalex.org/W2235890118","https://openalex.org/W2182505955"],"abstract_inverted_index":{"Regression":[0],"testing":[1,24,65],"is":[2,25,34],"a":[3,89],"crucial":[4],"part":[5],"of":[6,22,82,91,110,147],"software":[7,12],"development.":[8],"It":[9],"checks":[10],"that":[11,26,94,115,124,133],"changes":[13],"do":[14],"not":[15],"break":[16],"existing":[17],"functionality.":[18],"An":[19],"important":[20,145],"assumption":[21],"regression":[23,64],"test":[27,33,74],"outcomes":[28],"are":[29],"deterministic:":[30],"an":[31],"unmodified":[32],"expected":[35],"to":[36,71,127],"either":[37],"always":[38,41],"pass":[39],"or":[40],"fail":[42],"for":[43],"the":[44,63,78,105,144],"same":[45],"code":[46],"under":[47],"test.":[48],"Unfortunately,":[49],"in":[50,87,99],"practice,":[51],"some":[52],"tests":[53,61,98],"often":[54],"called":[55],"flaky":[56,83,97,111,118,129,149],"tests\u2014have":[57],"non-deterministic":[58],"outcomes.":[59],"Such":[60],"undermine":[62],"as":[66],"they":[67],"make":[68],"it":[69],"difficult":[70],"rely":[72],"on":[73,143],"results.":[75],"We":[76,85,103,131],"present":[77],"first":[79],"extensive":[80],"study":[81,86],"tests.":[84,130,150],"detail":[88],"total":[90],"201":[92],"commits":[93],"likely":[95],"fix":[96,128],"51":[100],"open-source":[101],"projects.":[102],"classify":[104],"most":[106],"common":[107,122],"root":[108],"causes":[109],"tests,":[112],"identify":[113],"approaches":[114],"could":[116],"manifest":[117],"behavior,":[119],"and":[120,136],"describe":[121],"strategies":[123],"developers":[125],"use":[126],"believe":[132],"our":[134],"insights":[135],"implications":[137],"can":[138],"help":[139],"guide":[140],"future":[141],"research":[142],"topic":[146],"(avoiding)":[148]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":44},{"year":2024,"cited_by_count":46},{"year":2023,"cited_by_count":38},{"year":2022,"cited_by_count":38},{"year":2021,"cited_by_count":71},{"year":2020,"cited_by_count":50},{"year":2019,"cited_by_count":51},{"year":2018,"cited_by_count":23},{"year":2017,"cited_by_count":16},{"year":2016,"cited_by_count":8},{"year":2015,"cited_by_count":12},{"year":2014,"cited_by_count":1}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}