{"id":"https://openalex.org/W4296872689","doi":"https://doi.org/10.1109/tse.2022.3208864","title":"Test Flakiness Across Programming Languages","display_name":"Test Flakiness Across Programming Languages","publication_year":2022,"publication_date":"2022-09-23","ids":{"openalex":"https://openalex.org/W4296872689","doi":"https://doi.org/10.1109/tse.2022.3208864"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2022.3208864","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2022.3208864","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102491923","display_name":"Keila Barbosa","orcid":null},"institutions":[{"id":"https://openalex.org/I25112270","display_name":"Universidade Federal de Pernambuco","ror":"https://ror.org/047908t24","country_code":"BR","type":"education","lineage":["https://openalex.org/I25112270"]}],"countries":["BR"],"is_corresponding":true,"raw_author_name":"Keila Barbosa","raw_affiliation_strings":["Federal University of Pernambuco, Recife, PE, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Federal University of Pernambuco, Recife, PE, Brazil","institution_ids":["https://openalex.org/I25112270"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037342003","display_name":"Ronivaldo Ferreira","orcid":"https://orcid.org/0000-0002-5812-8707"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ronivaldo Ferreira","raw_affiliation_strings":["Federal University of Par&#x00E1;, Bel&#x00E9;m, PA, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-5812-8707","affiliations":[{"raw_affiliation_string":"Federal University of Par&#x00E1;, Bel&#x00E9;m, PA, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016508154","display_name":"Gustavo Pinto","orcid":"https://orcid.org/0000-0001-7598-2799"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gustavo Pinto","raw_affiliation_strings":["Federal University of Par&#x00E1;, Bel&#x00E9;m, PA, Brazil"],"raw_orcid":"https://orcid.org/0000-0001-7598-2799","affiliations":[{"raw_affiliation_string":"Federal University of Par&#x00E1;, Bel&#x00E9;m, PA, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020353093","display_name":"Marcelo d\u2019Amorim","orcid":"https://orcid.org/0000-0002-1323-8769"},"institutions":[{"id":"https://openalex.org/I25112270","display_name":"Universidade Federal de Pernambuco","ror":"https://ror.org/047908t24","country_code":"BR","type":"education","lineage":["https://openalex.org/I25112270"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Marcelo d'Amorim","raw_affiliation_strings":["Federal University of Pernambuco, Recife, PE, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-1323-8769","affiliations":[{"raw_affiliation_string":"Federal University of Pernambuco, Recife, PE, Brazil","institution_ids":["https://openalex.org/I25112270"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064360478","display_name":"Breno Miranda","orcid":"https://orcid.org/0000-0001-9608-9393"},"institutions":[{"id":"https://openalex.org/I25112270","display_name":"Universidade Federal de Pernambuco","ror":"https://ror.org/047908t24","country_code":"BR","type":"education","lineage":["https://openalex.org/I25112270"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Breno Miranda","raw_affiliation_strings":["Federal University of Pernambuco, Recife, PE, Brazil"],"raw_orcid":"https://orcid.org/0000-0001-9608-9393","affiliations":[{"raw_affiliation_string":"Federal University of Pernambuco, Recife, PE, Brazil","institution_ids":["https://openalex.org/I25112270"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5102491923"],"corresponding_institution_ids":["https://openalex.org/I25112270"],"apc_list":null,"apc_paid":null,"fwci":4.1435,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.94408403,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"49","issue":"4","first_page":"2039","last_page":"2052"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.7517402172088623},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.7008771896362305},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6555390954017639},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.5892019271850586},{"id":"https://openalex.org/keywords/regression-testing","display_name":"Regression testing","score":0.4847395122051239},{"id":"https://openalex.org/keywords/notation","display_name":"Notation","score":0.45945388078689575},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.4535239636898041},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.42798614501953125},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.38191652297973633},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.31657910346984863},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.24808773398399353},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2204512655735016},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.13234007358551025},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.11863970756530762},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"Regression analysis","score":0.10706451535224915},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.09155312180519104}],"concepts":[{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.7517402172088623},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.7008771896362305},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6555390954017639},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.5892019271850586},{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.4847395122051239},{"id":"https://openalex.org/C45357846","wikidata":"https://www.wikidata.org/wiki/Q2001982","display_name":"Notation","level":2,"score":0.45945388078689575},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.4535239636898041},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.42798614501953125},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.38191652297973633},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.31657910346984863},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.24808773398399353},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2204512655735016},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.13234007358551025},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.11863970756530762},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.10706451535224915},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.09155312180519104}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2022.3208864","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2022.3208864","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.5400000214576721,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1984766364","https://openalex.org/W2021838450","https://openalex.org/W2026926213","https://openalex.org/W2054961017","https://openalex.org/W2129377409","https://openalex.org/W2149868499","https://openalex.org/W2150733339","https://openalex.org/W2759963882","https://openalex.org/W2767352876","https://openalex.org/W2786619380","https://openalex.org/W2794756405","https://openalex.org/W2809559400","https://openalex.org/W2885226523","https://openalex.org/W2898805736","https://openalex.org/W2901387825","https://openalex.org/W2901458396","https://openalex.org/W2951170114","https://openalex.org/W2955940099","https://openalex.org/W2957454295","https://openalex.org/W2968086426","https://openalex.org/W3091299963","https://openalex.org/W3095312267","https://openalex.org/W3102799430","https://openalex.org/W3106532184","https://openalex.org/W3110368543","https://openalex.org/W3157178205","https://openalex.org/W3162315425","https://openalex.org/W3163861420","https://openalex.org/W3164099559","https://openalex.org/W3167640545","https://openalex.org/W3169025994","https://openalex.org/W3181963574","https://openalex.org/W3193667668","https://openalex.org/W3195598538","https://openalex.org/W4233654496","https://openalex.org/W4233711156","https://openalex.org/W4245116308","https://openalex.org/W6685042213","https://openalex.org/W6752429367"],"related_works":["https://openalex.org/W2179621094","https://openalex.org/W1978406750","https://openalex.org/W2028796071","https://openalex.org/W2018145554","https://openalex.org/W2054959879","https://openalex.org/W3134448717","https://openalex.org/W2243231242","https://openalex.org/W2052414005","https://openalex.org/W2187337904","https://openalex.org/W2795302276"],"abstract_inverted_index":{"Regression":[0],"Testing":[1],"(RT)":[2],"is":[3,24,32,57,62,197],"a":[4,25,44,76,116,135],"quality-assurance":[5],"practice":[6],"commonly":[7],"adopted":[8],"in":[9,68,156,173,192],"the":[10,128,148],"software":[11,66],"industry":[12],"to":[13,34,184,199,211],"check":[14],"if":[15],"functionality":[16],"remains":[17],"intact":[18],"after":[19],"code":[20],"changes.":[21],"Test":[22],"flakiness":[23,51,64,172,182],"serious":[26],"problem":[27],"for":[28,65,96],"RT.":[29],"A":[30],"test":[31,50,63,84],"said":[33],"be":[35,113,132],"flaky":[36,212],"when":[37],"it":[38],"non-deterministically":[39],"passes":[40],"or":[41,231],"fails":[42],"on":[43,53,75,79],"fixed":[45,133],"environment.":[46],"Prior":[47],"work":[48],"studied":[49,103],"primarily":[52],"Java":[54],"programs.":[55],"It":[56],"unclear,":[58],"however,":[59],"how":[60],"problematic":[61],"written":[67],"other":[69],"programming":[70,99],"languages.":[71],"This":[72],"paper":[73],"reports":[74],"study":[77],"focusing":[78],"three":[80],"central":[81],"aspects":[82],"of":[83,119,127,139,147,171,187],"flakiness:":[85],"concentration,":[86,91],"similarity,":[87,151],"and":[88,108,130,159,165],"cost.":[89],"Considering":[90,150,204],"our":[92],"results":[93],"show":[94],"that,":[95],"any":[97],"given":[98],"language":[100],"that":[101,195,208],"we":[102,176,206],"(C,":[104],"Go,":[105],"Java,":[106],"JS,":[107],"Python),":[109],"most":[110,174],"issues":[111,209],"could":[112,131],"explained":[114],"by":[115,134],"small":[117,137],"fraction":[118,138],"root":[120,123,157],"causes":[121,124,158,170],"(5/13":[122],"cover":[125,145],"78.07%":[126],"issues)":[129],"relatively":[136],"fix":[140,143],"strategies":[141,144],"(10/23":[142],"85.20%":[146],"issues).":[149],"although":[152],"there":[153,196],"were":[154],"commonalities":[155],"fixes":[160],"across":[161],"languages":[162],"(e.g.,":[163,181],"concurrency":[164],"async":[166],"wait":[167],"are":[168,189,214,221],"common":[169,191],"languages),":[175],"also":[177],"found":[178,207],"important":[179],"differences":[180],"due":[183],"improper":[185],"release":[186],"resources":[188],"more":[190],"C),":[193],"suggesting":[194,229,239],"opportunity":[198],"fine":[200],"tuning":[201],"analysis":[202],"tools.":[203],"cost,":[205],"related":[210],"tests":[213],"resolved":[215],"either":[216],"very":[217,232],"early":[218],"once":[219],"they":[220],"posted":[222],"(":[223,234],"<inline-formula><tex-math":[224,235],"notation=\"LaTeX\">$&lt;":[225],"$</tex-math></inline-formula>":[226],"10":[227],"days),":[228,238],"relevance,":[230],"late":[233],"notation=\"LaTeX\">$&gt;$</tex-math></inline-formula>":[236],"100":[237],"irrelevance.":[240]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}