{"id":"https://openalex.org/W4400934506","doi":"https://doi.org/10.1145/3680463","title":"An Empirical Study of Testing Machine Learning in the Wild","display_name":"An Empirical Study of Testing Machine Learning in the Wild","publication_year":2024,"publication_date":"2024-07-24","ids":{"openalex":"https://openalex.org/W4400934506","doi":"https://doi.org/10.1145/3680463"},"language":"en","primary_location":{"id":"doi:10.1145/3680463","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3680463","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3680463","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3680463","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058935035","display_name":"Moses Openja","orcid":"https://orcid.org/0000-0002-4115-3834"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Moses Openja","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","Polytechnique Montr\u00e9al, Canada"],"raw_orcid":"https://orcid.org/0000-0002-4115-3834","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","institution_ids":["https://openalex.org/I45683168"]},{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I45683168"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071052367","display_name":"Foutse Khomh","orcid":"https://orcid.org/0000-0002-5704-4173"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Foutse Khomh","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","Polytechnique Montr\u00e9al, Canada"],"raw_orcid":"https://orcid.org/0000-0002-5704-4173","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","institution_ids":["https://openalex.org/I45683168"]},{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I45683168"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005515943","display_name":"Armstrong Foundjem","orcid":"https://orcid.org/0000-0001-6356-1938"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Armstrong Foundjem","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","Polytechnique Montr\u00e9al, Canada"],"raw_orcid":"https://orcid.org/0000-0001-6356-1938","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","institution_ids":["https://openalex.org/I45683168"]},{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I45683168"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085576468","display_name":"Zhen Ming Jiang","orcid":"https://orcid.org/0000-0002-3063-3197"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zhen Ming (Jack) Jiang","raw_affiliation_strings":["York University, Toronto, Ontario, Canada","York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)"],"raw_orcid":"https://orcid.org/0000-0002-3063-3197","affiliations":[{"raw_affiliation_string":"York University, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077300412","display_name":"Mouna Abidi","orcid":"https://orcid.org/0000-0002-9058-8519"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mouna Abidi","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","Polytechnique Montr\u00e9al, Canada"],"raw_orcid":"https://orcid.org/0000-0002-9058-8519","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Queb\u00e9c, Canada","institution_ids":["https://openalex.org/I45683168"]},{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I45683168"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091586373","display_name":"Ahmed E. Hassan","orcid":"https://orcid.org/0000-0001-7749-5513"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ahmed E. Hassan","raw_affiliation_strings":["Queen's University, Kingston, Ontario, Canada","Queen's University [Kingston, Canada] (99 University Ave, Kingston, ON K7L 3N6 - Canada)"],"raw_orcid":"https://orcid.org/0000-0001-7749-5513","affiliations":[{"raw_affiliation_string":"Queen's University, Kingston, Ontario, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Queen's University [Kingston, Canada] (99 University Ave, Kingston, ON K7L 3N6 - Canada)","institution_ids":["https://openalex.org/I204722609"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.5006,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.90490171,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"34","issue":"1","first_page":"1","last_page":"63"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8034570813179016},{"id":"https://openalex.org/keywords/white-box-testing","display_name":"White-box testing","score":0.622186541557312},{"id":"https://openalex.org/keywords/regression-testing","display_name":"Regression testing","score":0.598483145236969},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.5889098644256592},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.5828945636749268},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.5603055357933044},{"id":"https://openalex.org/keywords/quality-assurance","display_name":"Quality assurance","score":0.49586257338523865},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4914182424545288},{"id":"https://openalex.org/keywords/software-reliability-testing","display_name":"Software reliability testing","score":0.48171931505203247},{"id":"https://openalex.org/keywords/manual-testing","display_name":"Manual testing","score":0.479953795671463},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.4561774432659149},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4537896513938904},{"id":"https://openalex.org/keywords/non-regression-testing","display_name":"Non-regression testing","score":0.45256900787353516},{"id":"https://openalex.org/keywords/software-performance-testing","display_name":"Software performance testing","score":0.43886175751686096},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.42130953073501587},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41675513982772827},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.4124418795108795},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3892950713634491},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.376394122838974},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.3206337094306946},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.29550421237945557},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.17925989627838135},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.14900505542755127}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8034570813179016},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.622186541557312},{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.598483145236969},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.5889098644256592},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.5828945636749268},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.5603055357933044},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.49586257338523865},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4914182424545288},{"id":"https://openalex.org/C52928878","wikidata":"https://www.wikidata.org/wiki/Q7554226","display_name":"Software reliability testing","level":5,"score":0.48171931505203247},{"id":"https://openalex.org/C182122060","wikidata":"https://www.wikidata.org/wiki/Q6752328","display_name":"Manual testing","level":5,"score":0.479953795671463},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.4561774432659149},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4537896513938904},{"id":"https://openalex.org/C86469151","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Non-regression testing","level":5,"score":0.45256900787353516},{"id":"https://openalex.org/C178059732","wikidata":"https://www.wikidata.org/wiki/Q1982529","display_name":"Software performance testing","level":5,"score":0.43886175751686096},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.42130953073501587},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41675513982772827},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.4124418795108795},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3892950713634491},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.376394122838974},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.3206337094306946},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.29550421237945557},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.17925989627838135},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.14900505542755127},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3680463","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3680463","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3680463","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},{"id":"pmh:oai:publications.polymtl.ca:59775","is_oa":false,"landing_page_url":"https://publications.polymtl.ca/59775/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401013","display_name":"PolyPublie (\u00c9cole Polytechnique de Montr\u00e9al)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45683168","host_organization_name":"Polytechnique Montr\u00e9al","host_organization_lineage":["https://openalex.org/I45683168"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Article de revue"},{"id":"pmh:oai:HAL:hal-04661591v1","is_oa":false,"landing_page_url":"https://hal.science/hal-04661591","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM Transactions on Software Engineering and Methodology, 2024, ⟨10.1145/3680463⟩","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1145/3680463","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3680463","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3680463","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400934506.pdf"},"referenced_works_count":106,"referenced_works":["https://openalex.org/W1537202","https://openalex.org/W24642671","https://openalex.org/W317957491","https://openalex.org/W1488692668","https://openalex.org/W1548664830","https://openalex.org/W1797150484","https://openalex.org/W1961263830","https://openalex.org/W1964992116","https://openalex.org/W1994372357","https://openalex.org/W2001170097","https://openalex.org/W2004291985","https://openalex.org/W2013587512","https://openalex.org/W2014515160","https://openalex.org/W2026905436","https://openalex.org/W2029254184","https://openalex.org/W2032947860","https://openalex.org/W2046945713","https://openalex.org/W2077831477","https://openalex.org/W2104577574","https://openalex.org/W2107409339","https://openalex.org/W2109388592","https://openalex.org/W2115919801","https://openalex.org/W2118542736","https://openalex.org/W2119112357","https://openalex.org/W2122646361","https://openalex.org/W2128889010","https://openalex.org/W2137440139","https://openalex.org/W2162200351","https://openalex.org/W2168391307","https://openalex.org/W2169599971","https://openalex.org/W2230801863","https://openalex.org/W2248060815","https://openalex.org/W2277112320","https://openalex.org/W2282821441","https://openalex.org/W2295658119","https://openalex.org/W2559885217","https://openalex.org/W2592929672","https://openalex.org/W2603766943","https://openalex.org/W2606515170","https://openalex.org/W2612872092","https://openalex.org/W2616028256","https://openalex.org/W2658306063","https://openalex.org/W2730550703","https://openalex.org/W2740323046","https://openalex.org/W2774423163","https://openalex.org/W2782864149","https://openalex.org/W2799007037","https://openalex.org/W2799640043","https://openalex.org/W2885205917","https://openalex.org/W2900472033","https://openalex.org/W2905034244","https://openalex.org/W2921530211","https://openalex.org/W2922234936","https://openalex.org/W2955224284","https://openalex.org/W2962772482","https://openalex.org/W2962851801","https://openalex.org/W2963126845","https://openalex.org/W2963857521","https://openalex.org/W2965862774","https://openalex.org/W2967682612","https://openalex.org/W2989344603","https://openalex.org/W2996601440","https://openalex.org/W2999072931","https://openalex.org/W3004276184","https://openalex.org/W3004493192","https://openalex.org/W3007157104","https://openalex.org/W3011408464","https://openalex.org/W3032984651","https://openalex.org/W3081036064","https://openalex.org/W3087720408","https://openalex.org/W3095521706","https://openalex.org/W3098881644","https://openalex.org/W3102710196","https://openalex.org/W3121263626","https://openalex.org/W3123979795","https://openalex.org/W3139029978","https://openalex.org/W3139734284","https://openalex.org/W3165292502","https://openalex.org/W3176022264","https://openalex.org/W3181414820","https://openalex.org/W3193448347","https://openalex.org/W3211887526","https://openalex.org/W4210374324","https://openalex.org/W4213367174","https://openalex.org/W4220826582","https://openalex.org/W4236659663","https://openalex.org/W4240336314","https://openalex.org/W4244923041","https://openalex.org/W4246267392","https://openalex.org/W4252090733","https://openalex.org/W4253161820","https://openalex.org/W4253913023","https://openalex.org/W4280598010","https://openalex.org/W4281742620","https://openalex.org/W4282560685","https://openalex.org/W4283273467","https://openalex.org/W4300874380","https://openalex.org/W4312825166","https://openalex.org/W4312847312","https://openalex.org/W4381989586","https://openalex.org/W4386825342","https://openalex.org/W4389685461","https://openalex.org/W4390091644","https://openalex.org/W6629214716","https://openalex.org/W6799236523","https://openalex.org/W6803200579"],"related_works":["https://openalex.org/W3214776400","https://openalex.org/W3197709817","https://openalex.org/W2560445721","https://openalex.org/W3133844515","https://openalex.org/W2387992358","https://openalex.org/W4389884952","https://openalex.org/W2147394043","https://openalex.org/W2362944210","https://openalex.org/W2915670848","https://openalex.org/W2296409010"],"abstract_inverted_index":{"Background":[0],":":[1,123,159,213,328],"Recently,":[2],"machine":[3],"and":[4,151,175,196,207,233,244,273,278,286,294,304,323,341,344],"deep":[5],"learning":[6],"(ML/DL)":[7],"algorithms":[8],"have":[9,107],"been":[10],"increasingly":[11],"adopted":[12,99],"in":[13,71,100,138,205,283,298,346],"many":[14],"software":[15,37,80,168,210,338],"systems.":[16,211],"Due":[17],"to":[18,86,141,165,200,263],"their":[19,152,202],"inductive":[20],"nature,":[21],"ensuring":[22],"the":[23,33,48,51,119,130,139,143,148,155,190,334],"quality":[24,73],"of":[25,50,252,259,312,337],"these":[26,94],"systems":[27,38,58],"remains":[28],"a":[29,162],"significant":[30],"challenge":[31],"for":[32,227],"research":[34,70],"community.":[35],"Traditionally,":[36],"were":[39],"constructed":[40],"deductively,":[41],"by":[42],"writing":[43],"explicit":[44],"rules":[45,60],"that":[46],"govern":[47],"behavior":[49],"system":[52],"as":[53,83,237,316],"program":[54],"code.":[55],"However,":[56,89],"ML/DL":[57,72,180,209],"infer":[59],"from":[61,78,109,177],"training":[62],"data":[63],"i.e.,":[64],"they":[65],"are":[66,98,231,256,265],"generated":[67],"inductively.":[68],"Recent":[69],"assurance":[74],"has":[75],"adapted":[76],"concepts":[77],"traditional":[79],"testing,":[81,85],"such":[82,236,315],"mutation":[84],"improve":[87],"reliability.":[88],"it":[90],"is":[91,114,280,296],"unclear":[92],"if":[93,103],"proposed":[95],"testing":[96,105,120,137,149,169,191,198,224,313,339,349],"techniques":[97,340],"practice,":[101],"or":[102],"new":[104],"strategies":[106],"emerged":[108],"real-world":[110],"ML":[111,136,144,156,167,194,254,348],"deployments.":[112],"There":[113],"little":[115],"empirical":[116,133],"evidence":[117],"about":[118],"strategies.":[121],"Aims":[122],"To":[124],"fill":[125],"this":[126],"gap,":[127],"we":[128,187],"perform":[129],"first":[131],"fine-grained":[132],"study":[134,164,330],"on":[135,182,333],"wild":[140],"identify":[142],"properties":[145,255],"being":[146],"tested,":[147,257,267],"strategies,":[150,192,225],"implementation":[153],"throughout":[154],"workflow.":[157],"Method":[158],"We":[160,171,308],"conducted":[161],"mixed-methods":[163],"understand":[166,201],"practices.":[170,350],"analyzed":[172],"test":[173],"files":[174],"cases":[176],"11":[178],"open-source":[179],"projects":[181],"GitHub.":[183],"Using":[184],"open":[185],"coding,":[186],"manually":[188],"examined":[189],"tested":[193,282,297],"properties,":[195],"implemented":[197],"methods":[199],"practical":[203],"application":[204],"building":[206],"releasing":[208],"Results":[212],"Our":[214],"findings":[215],"reveal":[216],"several":[217],"key":[218],"insights:":[219],"(1)":[220],"The":[221],"most":[222],"common":[223],"accounting":[226],"less":[228],"than":[229],"40%,":[230],"Grey-box":[232],"White-box":[234],"methods,":[235,314],"Negative":[238],"Testing":[239,246,318,321,325],",":[240,243,270,272,319,322],"Oracle":[241],"Approximation":[242],"Statistical":[245],".":[247,275,326],"(2)":[248],"A":[249],"wide":[250],"range":[251],"\\(17\\)":[253],"out":[258],"which":[260],"only":[261],"20%":[262],"30%":[264],"frequently":[266],"including":[268],"Consistency":[269],"Correctness":[271],"Efficiency":[274],"(3)":[276],"Bias":[277],"Fairness":[279],"more":[281],"Recommendation":[284],"(6%)":[285],"Computer":[287],"Vision":[288],"(CV)":[289],"(3.9%)":[290],"systems,":[291],"while":[292],"Security":[293],"Privacy":[295],"CV":[299],"(2%),":[300],"Application":[301],"Platforms":[302],"(0.9%),":[303],"NLP":[305],"(0.5%).":[306],"(4)":[307],"identified":[309],"13":[310],"types":[311],"Unit":[317],"Input":[320],"Model":[324],"Conclusions":[327],"This":[329],"sheds":[331],"light":[332],"current":[335],"adoption":[336],"highlights":[342],"gaps":[343],"limitations":[345],"existing":[347]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":3}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}