{"id":"https://openalex.org/W3194175074","doi":"https://doi.org/10.1145/3468264.3468615","title":"FLEX: fixing flaky tests in machine learning projects by updating assertion bounds","display_name":"FLEX: fixing flaky tests in machine learning projects by updating assertion bounds","publication_year":2021,"publication_date":"2021-08-18","ids":{"openalex":"https://openalex.org/W3194175074","doi":"https://doi.org/10.1145/3468264.3468615","mag":"3194175074"},"language":"en","primary_location":{"id":"doi:10.1145/3468264.3468615","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3468264.3468615","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063258857","display_name":"Saikat Dutta","orcid":"https://orcid.org/0000-0001-6021-5407"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saikat Dutta","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081483504","display_name":"August Shi","orcid":"https://orcid.org/0000-0001-8239-3124"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"August Shi","raw_affiliation_strings":["University of Texas at Austin, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057462458","display_name":"Sa\u0161a Misailov\u00edc","orcid":"https://orcid.org/0000-0001-7319-8845"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sasa Misailovic","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.9114,"has_fulltext":false,"cited_by_count":31,"citation_normalized_percentile":{"value":0.97855649,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"603","last_page":"614"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/randomness","display_name":"Randomness","score":0.8225322961807251},{"id":"https://openalex.org/keywords/assertion","display_name":"Assertion","score":0.813675582408905},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8133009672164917},{"id":"https://openalex.org/keywords/flex","display_name":"FLEX","score":0.744701623916626},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6932251453399658},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.6343731880187988},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5733200907707214},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4858526885509491},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.45842257142066956},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39421188831329346},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3121275305747986},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.296888530254364},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09552904963493347}],"concepts":[{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.8225322961807251},{"id":"https://openalex.org/C40422974","wikidata":"https://www.wikidata.org/wiki/Q741248","display_name":"Assertion","level":2,"score":0.813675582408905},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8133009672164917},{"id":"https://openalex.org/C2776252893","wikidata":"https://www.wikidata.org/wiki/Q1364836","display_name":"FLEX","level":2,"score":0.744701623916626},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6932251453399658},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.6343731880187988},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5733200907707214},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4858526885509491},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.45842257142066956},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39421188831329346},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3121275305747986},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.296888530254364},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09552904963493347},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3468264.3468615","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3468264.3468615","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.41999998688697815,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G3365175841","display_name":null,"funder_award_id":"CCF-1846354,CCF-1956374,CCF-2008883,CCF-2028861","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307764","display_name":"Microsoft","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320308737","display_name":"Facebook","ror":"https://ror.org/01zbnvs85"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W129305155","https://openalex.org/W146863617","https://openalex.org/W246407485","https://openalex.org/W1526710119","https://openalex.org/W1533572394","https://openalex.org/W1582248076","https://openalex.org/W1601542853","https://openalex.org/W1890754682","https://openalex.org/W1984043109","https://openalex.org/W1995945562","https://openalex.org/W2026386347","https://openalex.org/W2026926213","https://openalex.org/W2036983007","https://openalex.org/W2042251585","https://openalex.org/W2049871268","https://openalex.org/W2054137409","https://openalex.org/W2078864692","https://openalex.org/W2097041275","https://openalex.org/W2138428785","https://openalex.org/W2158082837","https://openalex.org/W2217402295","https://openalex.org/W2287023446","https://openalex.org/W2295266283","https://openalex.org/W2477215876","https://openalex.org/W2577537660","https://openalex.org/W2578314903","https://openalex.org/W2786619380","https://openalex.org/W2790880830","https://openalex.org/W2859484040","https://openalex.org/W2894048303","https://openalex.org/W2897613819","https://openalex.org/W2899115773","https://openalex.org/W2899445138","https://openalex.org/W2919115771","https://openalex.org/W2951170114","https://openalex.org/W2951926492","https://openalex.org/W2953682625","https://openalex.org/W2954903132","https://openalex.org/W2957454295","https://openalex.org/W2962756933","https://openalex.org/W2964321317","https://openalex.org/W2967287319","https://openalex.org/W2968086426","https://openalex.org/W2970971581","https://openalex.org/W2995734336","https://openalex.org/W2999072931","https://openalex.org/W3000442532","https://openalex.org/W3040168066","https://openalex.org/W3043238989","https://openalex.org/W3109004940","https://openalex.org/W3109892451","https://openalex.org/W3123045479","https://openalex.org/W3162204369","https://openalex.org/W4212944460","https://openalex.org/W4229678607","https://openalex.org/W4239336062","https://openalex.org/W4310299640"],"related_works":["https://openalex.org/W4250698707","https://openalex.org/W4244545186","https://openalex.org/W4247748223","https://openalex.org/W1008394927","https://openalex.org/W183173419","https://openalex.org/W4253292776","https://openalex.org/W2483231410","https://openalex.org/W4402871433","https://openalex.org/W4244549413","https://openalex.org/W1991833525"],"abstract_inverted_index":{"Many":[0],"machine":[1],"learning":[2],"(ML)":[3],"algorithms":[4],"are":[5],"inherently":[6],"random":[7],"\u2013":[8],"multiple":[9],"executions":[10],"using":[11],"the":[12,43,52],"same":[13],"inputs":[14],"may":[15,60],"produce":[16],"slightly":[17],"different":[18],"results":[19,54],"each":[20],"time.":[21],"Randomness":[22],"impacts":[23],"how":[24],"developers":[25],"write":[26],"tests":[27],"that":[28],"check":[29],"for":[30,46],"end-to-end":[31],"quality":[32,49],"of":[33,36],"their":[34],"implementations":[35],"these":[37],"ML":[38],"algorithms.":[39],"In":[40],"particular,":[41],"selecting":[42],"proper":[44],"thresholds":[45],"comparing":[47],"obtained":[48],"metrics":[50],"with":[51],"reference":[53],"is":[55],"a":[56],"non-intuitive":[57],"task,":[58],"which":[59],"lead":[61],"to":[62],"flaky":[63],"test":[64],"executions.":[65]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}