{"id":"https://openalex.org/W4408791813","doi":"https://doi.org/10.1109/access.2025.3553626","title":"Understanding Flaky Tests Through Linguistic Diversity: A Cross-Language and Comparative Machine Learning Study","display_name":"Understanding Flaky Tests Through Linguistic Diversity: A Cross-Language and Comparative Machine Learning Study","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4408791813","doi":"https://doi.org/10.1109/access.2025.3553626"},"language":"en","primary_location":{"id":"doi:10.1109/access.2025.3553626","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3553626","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2025.3553626","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102717882","display_name":"Azeem Ahmad","orcid":"https://orcid.org/0000-0003-3049-1261"},"institutions":[{"id":"https://openalex.org/I142024983","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26","country_code":"SA","type":"education","lineage":["https://openalex.org/I142024983"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["ES","SA"],"is_corresponding":false,"raw_author_name":"Azeem Ahmad","raw_affiliation_strings":["Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","Department of Software Engineering, Prince Sultan University, Riyadh, KSA"],"raw_orcid":"https://orcid.org/0000-0003-3049-1261","affiliations":[{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","institution_ids":["https://openalex.org/I142024983"]},{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, KSA","institution_ids":["https://openalex.org/I142024983","https://openalex.org/I97750245"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xin Sun","orcid":"https://orcid.org/0009-0004-4854-6229"},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Xin Sun","raw_affiliation_strings":["Department of Computer Sciences, Linköping University, Linköping, Sweden"],"raw_orcid":"https://orcid.org/0009-0004-4854-6229","affiliations":[{"raw_affiliation_string":"Department of Computer Sciences, Linköping University, Linköping, Sweden","institution_ids":["https://openalex.org/I102134673"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003012052","display_name":"Muhammad Rashid Naeem","orcid":"https://orcid.org/0000-0003-2341-0443"},"institutions":[{"id":"https://openalex.org/I142024983","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26","country_code":"SA","type":"education","lineage":["https://openalex.org/I142024983"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["ES","SA"],"is_corresponding":false,"raw_author_name":"Muhammad Rashid Naeem","raw_affiliation_strings":["Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","Department of Software Engineering, Prince Sultan University, Riyadh, KSA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","institution_ids":["https://openalex.org/I142024983"]},{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, KSA","institution_ids":["https://openalex.org/I142024983","https://openalex.org/I97750245"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019308548","display_name":"Yasir Javed","orcid":"https://orcid.org/0000-0002-6311-027X"},"institutions":[{"id":"https://openalex.org/I142024983","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26","country_code":"SA","type":"education","lineage":["https://openalex.org/I142024983"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["ES","SA"],"is_corresponding":false,"raw_author_name":"Yasir Javed","raw_affiliation_strings":["Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","Department of Software Engineering, Prince Sultan University, Riyadh, KSA"],"raw_orcid":"https://orcid.org/0000-0002-6311-027X","affiliations":[{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","institution_ids":["https://openalex.org/I142024983"]},{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, KSA","institution_ids":["https://openalex.org/I142024983","https://openalex.org/I97750245"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041474454","display_name":"Mohammed Akour","orcid":"https://orcid.org/0000-0002-5858-8957"},"institutions":[{"id":"https://openalex.org/I142024983","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26","country_code":"SA","type":"education","lineage":["https://openalex.org/I142024983"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["ES","SA"],"is_corresponding":false,"raw_author_name":"Mohammad Akour","raw_affiliation_strings":["Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","Department of Software Engineering, Prince Sultan University, Riyadh, KSA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, Saudi Arabia","institution_ids":["https://openalex.org/I142024983"]},{"raw_affiliation_string":"Department of Software Engineering, Prince Sultan University, Riyadh, KSA","institution_ids":["https://openalex.org/I142024983","https://openalex.org/I97750245"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076739512","display_name":"Kristian Sandahl","orcid":"https://orcid.org/0000-0002-3052-5604"},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Kristian Sandahl","raw_affiliation_strings":["Department of Computer Sciences, Linköping University, Linköping, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-3052-5604","affiliations":[{"raw_affiliation_string":"Department of Computer Sciences, Linköping University, Linköping, Sweden","institution_ids":["https://openalex.org/I102134673"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":5.2763,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.9479638,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"13","issue":null,"first_page":"54561","last_page":"54584"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.573199987411499,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.573199987411499,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7408705353736877},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.5618748664855957},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.549447238445282},{"id":"https://openalex.org/keywords/linguistic-diversity","display_name":"Linguistic diversity","score":0.5267062783241272},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.4883422553539276},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46428337693214417},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.12576881051063538}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7408705353736877},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.5618748664855957},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.549447238445282},{"id":"https://openalex.org/C2992249680","wikidata":"https://www.wikidata.org/wiki/Q315","display_name":"Linguistic diversity","level":2,"score":0.5267062783241272},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.4883422553539276},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46428337693214417},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.12576881051063538},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2025.3553626","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3553626","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:DiVA.org:liu-213193","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-213193","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:doaj.org/article:d1a1475af3614a6dbbc8012c157eef92","is_oa":true,"landing_page_url":"https://doaj.org/article/d1a1475af3614a6dbbc8012c157eef92","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 13, Pp 54561-54584 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2025.3553626","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3553626","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.75,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320327702","display_name":"Prince Sultan University","ror":"https://ror.org/053mqrf26"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W2001619934","https://openalex.org/W2026926213","https://openalex.org/W2040870580","https://openalex.org/W2122111042","https://openalex.org/W2158698691","https://openalex.org/W2548334485","https://openalex.org/W2740295334","https://openalex.org/W2741220297","https://openalex.org/W2786619380","https://openalex.org/W2794756405","https://openalex.org/W2885226523","https://openalex.org/W2898805736","https://openalex.org/W2901458396","https://openalex.org/W2903950532","https://openalex.org/W2960013457","https://openalex.org/W2963935794","https://openalex.org/W3091299963","https://openalex.org/W3175011188","https://openalex.org/W4213009331","https://openalex.org/W4233152543","https://openalex.org/W4236137412","https://openalex.org/W4239510810","https://openalex.org/W4245429230","https://openalex.org/W4400762160","https://openalex.org/W4401880094","https://openalex.org/W6611762666","https://openalex.org/W6659397378","https://openalex.org/W6682304300","https://openalex.org/W6763151734","https://openalex.org/W6787344654","https://openalex.org/W6794287823"],"related_works":["https://openalex.org/W4236583520","https://openalex.org/W2322261865","https://openalex.org/W4398861705","https://openalex.org/W2734736160","https://openalex.org/W2409468626","https://openalex.org/W4206863193","https://openalex.org/W4205351689","https://openalex.org/W4283518176","https://openalex.org/W2936964361","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Software":[0],"development":[1],"is":[2,278],"significantly":[3],"impeded":[4],"by":[5,309],"flaky":[6,101,205,256,332],"tests,":[7],"which":[8,65,220],"intermittently":[9],"pass":[10],"or":[11,32],"fail":[12],"without":[13],"requiring":[14],"code":[15,262],"modifications,":[16],"resulting":[17],"in":[18,21,23,99,123,174,182,201,231,254,264,294,334],"a":[19,73,93,148,155,208,216,238,335],"decline":[20],"confidence":[22],"automated":[24,299],"testing":[25],"frameworks.":[26],"Code":[27],"smells":[28],"(i.e.,":[29],"test":[30,40,49,142],"case":[31],"production":[33],"code)":[34],"are":[35,267],"the":[36,46,89,105,133,139,186,202,225,246,287,296,305,329],"primary":[37],"cause":[38],"of":[39,48,75,92,95,107,135,141,157,176,193,204,210,227,233,240,289,298,331,337],"flakiness.":[41],"In":[42,129],"order":[43,130],"to":[44,131,223,275,328],"ascertain":[45,132],"prevalence":[47],"smells,":[50],"researchers":[51,320],"and":[52,84,120,126,151,167,179,199,235,251,260,291,312,319],"practitioners":[53],"have":[54],"examined":[55],"numerous":[56],"programming":[57,70,76,241],"languages.":[58,158,242],"However,":[59],"one":[60,69],"isolated":[61],"experiment":[62,261],"was":[63],"conducted,":[64],"focused":[66],"solely":[67],"on":[68,138,147,154],"language.":[71],"Across":[72],"variety":[74,94,156,239,336],"languages,":[77],"such":[78,109],"as":[79,110,207],"Java,":[80],"Python,":[81],"C++,":[82],"Go,":[83],"JavaScript,":[85],"this":[86,265],"study":[87,266],"examines":[88],"predictive":[90],"accuracy":[91,200,234],"machine":[96],"learning":[97],"classifiers":[98,108,173,250,293],"identifying":[100],"tests.":[102,257],"We":[103,214],"compare":[104],"performance":[106,229],"Random":[111,165],"Forest,":[112],"Decision":[113],"Tree,":[114],"Naive":[115],"Bayes,":[116],"Support":[117],"Vector":[118],"Machine,":[119],"Logistic":[121,168],"Regression":[122,169],"both":[124],"single-language":[125],"cross-language":[127,183],"settings.":[128],"impact":[134],"linguistic":[136],"diversity":[137],"flakiness":[140],"cases,":[143],"models":[144],"were":[145],"trained":[146],"single":[149],"language":[150],"subsequently":[152],"tested":[153],"The":[159,258],"following":[160],"key":[161],"findings":[162,190],"indicate":[163],"that":[164,324],"Forest":[166],"consistently":[170],"outperform":[171],"other":[172],"terms":[175,232],"accuracy,":[177],"adaptability,":[178],"generalizability,":[180],"particularly":[181],"environments.":[184,339],"Additionally,":[185],"investigation":[187],"contrasts":[188],"our":[189],"with":[191,321],"those":[192],"previous":[194],"research,":[195],"exhibiting":[196],"enhanced":[197],"precision":[198],"identification":[203],"tests":[206,333],"result":[209],"meticulous":[211],"classifier":[212,228],"selection.":[213],"conducted":[215],"thorough":[217],"statistical":[218],"analysis,":[219],"included":[221],"t-tests,":[222],"assess":[224],"importance":[226],"differences":[230],"F1-score":[236],"across":[237],"This":[243,315],"analysis":[244],"emphasizes":[245],"substantial":[247],"discrepancies":[248],"between":[249],"their":[252],"effectiveness":[253,288],"detecting":[255],"datasets":[259],"utilized":[263],"accessible":[268],"through":[269],"an":[270],"open":[271],"source":[272],"GitHub":[273],"repository":[274],"facilitate":[276],"reproducibility":[277],"available":[279],"at:":[280],"https://github.com/PELAB-LiU/FlakyCrossLanguage.":[283],"Our":[284],"results":[285],"emphasize":[286],"probabilistic":[290],"ensemble":[292],"improving":[295],"reliability":[297],"testing,":[300],"despite":[301],"certain":[302],"constraints,":[303],"including":[304],"potential":[306],"biases":[307],"introduced":[308],"language-specific":[310],"structures":[311],"dataset":[313],"variability.":[314],"research":[316],"provides":[317],"developers":[318],"practical":[322],"insights":[323],"can":[325],"be":[326],"applied":[327],"mitigation":[330],"software":[338]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}