{"id":"https://openalex.org/W4401908307","doi":"https://doi.org/10.1109/icst60714.2024.00037","title":"Cost of Flaky Tests in Continuous Integration: An Industrial Case Study","display_name":"Cost of Flaky Tests in Continuous Integration: An Industrial Case Study","alternate_titles":[],"publication_year":2024,"publication_date":"2024-05-27","ids":{"openalex":"https://openalex.org/W4401908307","doi":"https://doi.org/10.1109/icst60714.2024.00037"},"language":"en","primary_location":{"id":"doi:10.1109/icst60714.2024.00037","doi":"https://doi.org/10.1109/icst60714.2024.00037","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst60714.2024.00037","pdf_url":null,"source":{"id":"https://openalex.org/S7407085955","display_name":"IEEE International Conference on Software Testing, Verification and Validation (ICST)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4401908307"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://mediatum.ub.tum.de/1730194","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082494668","display_name":"Fabian Leinen","orcid":"https://orcid.org/0009-0007-9291-3315","observed_orcids":["https://orcid.org/0009-0007-9291-3315"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Fabian Leinen","raw_affiliation_strings":["Technical University of Munich,Munich,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich,Munich,Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005796935","display_name":"Daniel Elsner","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Daniel Elsner","raw_affiliation_strings":["Technical University of Munich,Munich,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich,Munich,Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002011805","display_name":"Alexander Pretschner","orcid":"https://orcid.org/0000-0002-5573-1201","observed_orcids":["https://orcid.org/0000-0002-5573-1201"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Alexander Pretschner","raw_affiliation_strings":["Technical University of Munich,Munich,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Munich,Munich,Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016649149","display_name":"Andreas Stahlbauer","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Andreas Stahlbauer","raw_affiliation_strings":["CQSE GmbH,Munich,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CQSE GmbH,Munich,Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054960268","display_name":"Michael Sailer","orcid":"https://orcid.org/0009-0001-4125-4942","observed_orcids":["https://orcid.org/0009-0001-4125-4942"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Michael Sailer","raw_affiliation_strings":["CQSE GmbH,Munich,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CQSE GmbH,Munich,Germany","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036232169","display_name":"Elmar J\u00fcrgens","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Elmar J\u00fcrgens","raw_affiliation_strings":["CQSE GmbH,Munich,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CQSE GmbH,Munich,Germany","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.839,"has_fulltext":false,"cited_by_count":23,"reviewed_by_count":0,"citation_normalized_percentile":{"value":0.95648078,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":92,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"329","last_page":"340"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7490000128746033,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7490000128746033,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.23319999873638153,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.009600000455975533,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/flaky-tests","display_name":"flaky tests","score":1.0},{"id":"https://openalex.org/keywords/continuous-integration","display_name":"continuous integration","score":0.9879999756813049},{"id":"https://openalex.org/keywords/cost-model","display_name":"cost model","score":0.7689999938011169},{"id":"https://openalex.org/keywords/developer-effort","display_name":"developer effort","score":0.5659999847412109},{"id":"https://openalex.org/keywords/test-repair","display_name":"test repair","score":0.8859999775886536},{"id":"https://openalex.org/keywords/software-testing-cost","display_name":"software testing cost","score":0.3889999985694885}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5015368461608887}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icst60714.2024.00037","doi":"https://doi.org/10.1109/icst60714.2024.00037","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst60714.2024.00037","pdf_url":null,"source":{"id":"https://openalex.org/S7407085955","display_name":"IEEE International Conference on Software Testing, Verification and Validation (ICST)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"},{"id":"pmh:oai:mediatum.ub.tum.de:node/1730194","doi":null,"is_oa":true,"landing_page_url":"https://mediatum.ub.tum.de/1730194","pdf_url":null,"source":{"id":"https://openalex.org/S4306400453","display_name":"mediaTUM (Technical University of Munich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I62916508","host_organization_name":"Technical University of Munich","host_organization_lineage":["https://openalex.org/I62916508"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:mediatum.ub.tum.de:node/1730194","doi":null,"is_oa":true,"landing_page_url":"https://mediatum.ub.tum.de/1730194","pdf_url":null,"source":{"id":"https://openalex.org/S4306400453","display_name":"mediaTUM (Technical University of Munich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I62916508","host_organization_name":"Technical University of Munich","host_organization_lineage":["https://openalex.org/I62916508"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":64,"referenced_works":["https://openalex.org/W1928770023","https://openalex.org/W1991292921","https://openalex.org/W2010985317","https://openalex.org/W2014515160","https://openalex.org/W2026926213","https://openalex.org/W2107500604","https://openalex.org/W2129377409","https://openalex.org/W2154941560","https://openalex.org/W2591823109","https://openalex.org/W2794756405","https://openalex.org/W2809559400","https://openalex.org/W2888303318","https://openalex.org/W2898805736","https://openalex.org/W2901387825","https://openalex.org/W2901458396","https://openalex.org/W2954297335","https://openalex.org/W2955940099","https://openalex.org/W2957454295","https://openalex.org/W2968086426","https://openalex.org/W3025671512","https://openalex.org/W3033013930","https://openalex.org/W3040168066","https://openalex.org/W3090144865","https://openalex.org/W3095312267","https://openalex.org/W3097014822","https://openalex.org/W3102799430","https://openalex.org/W3110368543","https://openalex.org/W3163861420","https://openalex.org/W3167640545","https://openalex.org/W3177739655","https://openalex.org/W3180921142","https://openalex.org/W3194175074","https://openalex.org/W3194600864","https://openalex.org/W3210114756","https://openalex.org/W4210788642","https://openalex.org/W4226137778","https://openalex.org/W4229843836","https://openalex.org/W4232064490","https://openalex.org/W4233376690","https://openalex.org/W4239672721","https://openalex.org/W4245116308","https://openalex.org/W4248799824","https://openalex.org/W4252613971","https://openalex.org/W4281892382","https://openalex.org/W4282569378","https://openalex.org/W4282842157","https://openalex.org/W4283073622","https://openalex.org/W4283074706","https://openalex.org/W4283080616","https://openalex.org/W4285162211","https://openalex.org/W4296872689","https://openalex.org/W4298519450","https://openalex.org/W4300420355","https://openalex.org/W4312416970","https://openalex.org/W4312429593","https://openalex.org/W4312656090","https://openalex.org/W4321593773","https://openalex.org/W4367316222","https://openalex.org/W4378364616","https://openalex.org/W4378418573","https://openalex.org/W4384009631","https://openalex.org/W4405602283","https://openalex.org/W4405602523","https://openalex.org/W6677671109"],"related_works":["https://openalex.org/W4402571013","https://openalex.org/W7117104612","https://openalex.org/W2794756405","https://openalex.org/W4367316222","https://openalex.org/W3040168066","https://openalex.org/W7015717045","https://openalex.org/W7204770234","https://openalex.org/W7005405669","https://openalex.org/W2957454295","https://openalex.org/W4416024585"],"abstract_inverted_index":{"Researchers":[0],"and":[1,26,40,73,75,106,129,191,216,228,261,282],"practitioners":[2,272],"alike":[3],"increasingly":[4],"often":[5],"perceive":[6],"flaky":[7,32,47,66,140,150,172,185,196,277,300],"tests":[8,48,67,151,186,197,212,278],"as":[9],"a":[10,18,77,94,98,220,236,291],"major":[11],"challenge":[12],"in":[13,50,68,93,152,235,241,279],"software":[14,101],"engineering.":[15],"They":[16],"spend":[17],"lot":[19],"of":[20,31,46,97,113,160,179,276,299],"effort":[21,166,258],"trying":[22],"to":[23,41,133,194,202,213,253,256,263,294,306],"detect,":[24],"repair,":[25],"mitigate":[27],"the":[28,44,85,122,135,145,153,161,180,208,231,254,274,297],"negative":[29],"effects":[30],"tests.":[33,141,266],"However,":[34],"it":[35],"is":[36,167],"yet":[37],"unclear":[38],"where":[39],"what":[42],"extent":[43],"costs":[45,86,222,298],"manifest":[49],"industrial":[51],"Continuous":[52],"Integration":[53],"(CI)":[54],"development":[55,70,114],"processes.":[56],"In":[57],"this":[58,91],"study,":[59],"we":[60,206],"compile":[61],"cost":[62,78,136,209,268,275],"factors":[63,137],"introduced":[64],"by":[65,139],"CI":[69,117],"from":[71,121,247,259],"research":[72],"practice":[74],"derive":[76],"model":[79,92,269],"that":[80,144],"allows":[81],"gaining":[82],"insight":[83],"into":[84,169],"incurred.":[87],"We":[88,109,142],"then":[89],"instantiate":[90],"case":[95,249,288],"study":[96,250,289],"large,":[99],"commercial":[100],"project":[102,155],"with":[103,149],"~30":[104],"developers":[105],"~1M":[107],"SLoC.":[108],"analyze":[110,273],"five":[111],"years":[112],"history,":[115],"including":[116],"test":[118,173,221],"logs,":[119],"commits":[120],"Version":[123],"Control":[124],"System":[125],"(VCS),":[126],"issue":[127],"tickets,":[128],"tracked":[130],"work":[131],"time":[132,146,182],"quantify":[134],"implied":[138],"find":[143,207],"spent":[147],"dealing":[148],"studied":[154],"represents":[156],"at":[157],"least":[158],"2.5%":[159],"productive":[162],"developer":[163],"time.":[164],"This":[165],"divided":[168],"investigating":[170],"potentially":[171],"failures,":[174],"which":[175,302],"accounts":[176],"for":[177,210],"1.1%":[178],"total":[181],"spent,":[183],"repairing":[184],"adds":[187,198],"another":[188],"1.3":[189],"%,":[190],"developing":[192],"tools":[193],"monitor":[195],"0.1":[199],"%.":[200],"Contrary":[201],"most":[203],"other":[204],"studies,":[205],"rerunning":[211,219,227,265],"be":[214],"negligible":[215],"inexpensive.":[217],"Automatically":[218],"0.02":[223],"cents,":[224],"while":[225],"not":[226],"thus":[229],"letting":[230],"pipeline":[232],"fail":[233],"results":[234],"manual":[237],"investigation":[238,260],"costing":[239],"$5.67":[240],"our":[242,248,287],"context.":[243],"The":[244],"insights":[245],"gained":[246],"have":[251],"led":[252],"decision":[255],"shift":[257],"repair":[262],"automatically":[264],"Our":[267],"can":[270,303],"help":[271],"their":[280],"context":[281],"make":[283],"informed":[284],"decisions.":[285],"Furthermore,":[286],"provides":[290],"first":[292],"step":[293],"better":[295],"understand":[296],"tests,":[301],"lead":[304],"researchers":[305],"industry-relevant":[307],"problems.":[308]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4401908307","counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2024-08-28T00:00:00"}
