{"id":"https://openalex.org/W4391558644","doi":"https://doi.org/10.1145/3597503.3608138","title":"Do Automatic Test Generation Tools Generate Flaky Tests?","display_name":"Do Automatic Test Generation Tools Generate Flaky Tests?","publication_year":2024,"publication_date":"2024-02-06","ids":{"openalex":"https://openalex.org/W4391558644","doi":"https://doi.org/10.1145/3597503.3608138"},"language":"en","primary_location":{"id":"doi:10.1145/3597503.3608138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3608138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3608138","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3608138","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074868913","display_name":"Martin Gruber","orcid":"https://orcid.org/0000-0003-0447-1224"},"institutions":[{"id":"https://openalex.org/I1283382300","display_name":"BMW (Germany)","ror":"https://ror.org/05vs9tj88","country_code":"DE","type":"company","lineage":["https://openalex.org/I1283382300","https://openalex.org/I4210156768"]},{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Martin Gruber","raw_affiliation_strings":["Chair of Software Engineering II, University of Passau, Passau, Germany","Developer Tools and Continuous Integration, BMW Group, Munich, Germany","Developer Tools and Continuous Integration, BMW Group, Munich, Germany Chair of Software Engineering II, University of Passau, Passau, Germany"],"affiliations":[{"raw_affiliation_string":"Chair of Software Engineering II, University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]},{"raw_affiliation_string":"Developer Tools and Continuous Integration, BMW Group, Munich, Germany","institution_ids":["https://openalex.org/I1283382300"]},{"raw_affiliation_string":"Developer Tools and Continuous Integration, BMW Group, Munich, Germany Chair of Software Engineering II, University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057159721","display_name":"Muhammad Firhard Roslan","orcid":"https://orcid.org/0009-0002-7177-1702"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Muhammad Firhard Roslan","raw_affiliation_strings":["Testing Research Group, University of Sheffield, Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Testing Research Group, University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038581120","display_name":"Owain Parry","orcid":"https://orcid.org/0000-0002-0917-1274"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Owain Parry","raw_affiliation_strings":["Testing Research Group, University of Sheffield, Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Testing Research Group, University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093047692","display_name":"Fabian Scharnb\u00f6ck","orcid":"https://orcid.org/0009-0007-1823-6985"},"institutions":[{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Fabian Scharnb\u00f6ck","raw_affiliation_strings":["Chair of Software Engineering II, University of Passau, Passau, Germany"],"affiliations":[{"raw_affiliation_string":"Chair of Software Engineering II, University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078598020","display_name":"Phil McMinn","orcid":"https://orcid.org/0000-0001-9137-7433"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Phil McMinn","raw_affiliation_strings":["Testing Research Group, University of Sheffield, Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Testing Research Group, University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079261847","display_name":"Gordon Fraser","orcid":"https://orcid.org/0000-0002-4364-6595"},"institutions":[{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gordon Fraser","raw_affiliation_strings":["Chair of Software Engineering II, University of Passau, Passau, Germany"],"affiliations":[{"raw_affiliation_string":"Chair of Software Engineering II, University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5074868913"],"corresponding_institution_ids":["https://openalex.org/I1283382300","https://openalex.org/I186354981"],"apc_list":null,"apc_paid":null,"fwci":5.6164,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.96336162,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"12"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11032","display_name":"VLSI and Analog Circuit Testing","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5950116515159607},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.48052486777305603},{"id":"https://openalex.org/keywords/automatic-test-pattern-generation","display_name":"Automatic test pattern generation","score":0.4402427077293396},{"id":"https://openalex.org/keywords/automatic-test-equipment","display_name":"Automatic test equipment","score":0.4325544238090515},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.32420802116394043},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.15467330813407898},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.08191677927970886},{"id":"https://openalex.org/keywords/electronic-circuit","display_name":"Electronic circuit","score":0.061399996280670166}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5950116515159607},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.48052486777305603},{"id":"https://openalex.org/C17626397","wikidata":"https://www.wikidata.org/wiki/Q837455","display_name":"Automatic test pattern generation","level":3,"score":0.4402427077293396},{"id":"https://openalex.org/C141842801","wikidata":"https://www.wikidata.org/wiki/Q363815","display_name":"Automatic test equipment","level":3,"score":0.4325544238090515},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.32420802116394043},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.15467330813407898},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.08191677927970886},{"id":"https://openalex.org/C134146338","wikidata":"https://www.wikidata.org/wiki/Q1815901","display_name":"Electronic circuit","level":2,"score":0.061399996280670166},{"id":"https://openalex.org/C51234621","wikidata":"https://www.wikidata.org/wiki/Q2149495","display_name":"Testability","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3597503.3608138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3608138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3608138","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3597503.3608138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3608138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3608138","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2292826426","display_name":"DTP 2018-19 University of Sheffield","funder_award_id":"EP/R513313/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G3499525715","display_name":null,"funder_award_id":"EP/R513","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G458780037","display_name":null,"funder_award_id":"Doctoral Training Partnership","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G6743243744","display_name":null,"funder_award_id":"unknown","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G8085335477","display_name":"Test FLARE (Test Flakiness Automated Reproduction and Explanation)","funder_award_id":"EP/X024539/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8452545418","display_name":null,"funder_award_id":"unknown","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8770309392","display_name":null,"funder_award_id":"EP/X024539/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391558644.pdf","grobid_xml":"https://content.openalex.org/works/W4391558644.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W1971650562","https://openalex.org/W2098413804","https://openalex.org/W2100987397","https://openalex.org/W2114869486","https://openalex.org/W2135841285","https://openalex.org/W2561675875","https://openalex.org/W2807453842","https://openalex.org/W2888533831","https://openalex.org/W2946788984","https://openalex.org/W2951170114","https://openalex.org/W2955940099","https://openalex.org/W2957454295","https://openalex.org/W2968086426","https://openalex.org/W3040168066","https://openalex.org/W3105684902","https://openalex.org/W3106532184","https://openalex.org/W3185773251","https://openalex.org/W3210114756","https://openalex.org/W4281672691","https://openalex.org/W4281892382","https://openalex.org/W4282842157","https://openalex.org/W4284703142","https://openalex.org/W4284705801","https://openalex.org/W4299301436","https://openalex.org/W4315836284","https://openalex.org/W4319150622","https://openalex.org/W4384009815","https://openalex.org/W4394109690","https://openalex.org/W6968978714"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W4254560580","https://openalex.org/W2131559056","https://openalex.org/W2127167802","https://openalex.org/W2080984854","https://openalex.org/W128516171","https://openalex.org/W1863819993","https://openalex.org/W2144004661","https://openalex.org/W4252286421","https://openalex.org/W2129020400"],"abstract_inverted_index":{"Non-deterministic":[0],"test":[1,43,72,99,207,226,229],"behavior,":[2],"or":[3,82,228],"flakiness,":[4],"is":[5,111,159],"common":[6,115],"and":[7,16,36,58,93,96,169,192,217],"dreaded":[8],"among":[9],"developers.":[10],"Researchers":[11],"have":[12],"studied":[13],"the":[14,23,147,174,199],"issue":[15,136],"proposed":[17],"approaches":[18],"to":[19,143,189,205],"mitigate":[20],"it.":[21],"However,":[22],"vast":[24],"majority":[25],"of":[26,38,149],"previous":[27],"work":[28],"has":[29],"only":[30],"considered":[31],"developer-written":[32,63,121,144],"tests.":[33,122],"The":[34],"prevalence":[35],"nature":[37],"flaky":[39,56,71,140,145,151,176],"tests":[40,57,89,118,152,177],"produced":[41],"by":[42,163,167],"generation":[44,208],"tools":[45,53],"remain":[46],"largely":[47],"unknown.":[48],"We":[49,74],"ask":[50],"whether":[51],"such":[52],"also":[54],"produce":[55],"how":[59],"these":[60,215],"differ":[61,178],"from":[62,180],"ones.":[64],"Furthermore,":[65],"we":[66,87],"evaluate":[67],"mechanisms":[68,127],"that":[69,109],"suppress":[70],"generation.":[73,230],"sample":[75],"6":[76],"356":[77],"projects":[78],"written":[79],"in":[80,116,120,129,133,225],"Java":[81],"Python.":[83],"For":[84],"each":[85,98],"project,":[86],"generate":[88],"using":[90,214],"EvoSuite":[91,130],"(Java)":[92],"Pynguin":[94],"(Python),":[95],"execute":[97],"200":[100],"times,":[101],"looking":[102],"for":[103,212,222],"inconsistent":[104],"outcomes.":[105],"Our":[106],"results":[107],"show":[108],"flakiness":[110,125,172,182,227],"at":[112],"least":[113],"as":[114,119,219],"generated":[117,150],"Nevertheless,":[123],"existing":[124],"suppression":[126],"implemented":[128],"are":[131,153,187],"effective":[132],"alleviating":[134],"this":[135],"(71.7":[137],"%":[138],"fewer":[139],"tests).":[141],"Compared":[142],"tests,":[146],"causes":[148],"distributed":[154],"differently.":[155],"Their":[156],"non-deterministic":[157],"behavior":[158],"more":[160],"frequently":[161],"caused":[162],"randomness,":[164],"rather":[165],"than":[166],"networking":[168],"concurrency.":[170],"Using":[171],"suppression,":[173],"remaining":[175],"significantly":[179],"any":[181],"previously":[183],"reported,":[184],"where":[185],"most":[186],"attributable":[188],"runtime":[190],"optimizations":[191],"EvoSuite-internal":[193],"resource":[194],"thresholds.":[195],"These":[196],"insights,":[197],"with":[198],"accompanying":[200],"dataset,":[201],"can":[202],"help":[203],"maintainers":[204],"improve":[206],"tools,":[209,216],"give":[210],"recommendations":[211],"developers":[213],"serve":[218],"a":[220],"foundation":[221],"future":[223],"research":[224]},"counts_by_year":[{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":4}],"updated_date":"2026-04-18T07:56:08.524223","created_date":"2025-10-10T00:00:00"}