{"id":"https://openalex.org/W7160906790","doi":"https://doi.org/10.1016/j.jss.2026.112925","title":"Understanding Machine Learning testing in practice","display_name":"Understanding Machine Learning testing in practice","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7160906790","doi":"https://doi.org/10.1016/j.jss.2026.112925"},"language":"en","primary_location":{"id":"doi:10.1016/j.jss.2026.112925","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2026.112925","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.jss.2026.112925","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107130597","display_name":"Alfonso Cannavale","orcid":"https://orcid.org/0009-0002-0209-5974"},"institutions":[{"id":"https://openalex.org/I131729948","display_name":"University of Salerno","ror":"https://ror.org/0192m2k53","country_code":"IT","type":"education","lineage":["https://openalex.org/I131729948"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Alfonso Cannavale","raw_affiliation_strings":["Software Engineering (SeSa) Lab - Department of Computer Science, University of Salerno, Italy"],"raw_orcid":"https://orcid.org/0009-0002-0209-5974","affiliations":[{"raw_affiliation_string":"Software Engineering (SeSa) Lab - Department of Computer Science, University of Salerno, Italy","institution_ids":["https://openalex.org/I131729948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053602609","display_name":"Valeria Pontillo","orcid":"https://orcid.org/0000-0001-6012-9947"},"institutions":[{"id":"https://openalex.org/I4210150763","display_name":"Gran Sasso Science Institute","ror":"https://ror.org/043qcb444","country_code":"IT","type":"education","lineage":["https://openalex.org/I160013858","https://openalex.org/I4210150763"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Valeria Pontillo","raw_affiliation_strings":["Gran Sasso Science Institute (GSSI), L\u2019Aquila, Italy"],"raw_orcid":"https://orcid.org/0000-0001-6012-9947","affiliations":[{"raw_affiliation_string":"Gran Sasso Science Institute (GSSI), L\u2019Aquila, Italy","institution_ids":["https://openalex.org/I4210150763"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079088548","display_name":"Andrea De Lucia","orcid":"https://orcid.org/0000-0002-4238-1425"},"institutions":[{"id":"https://openalex.org/I131729948","display_name":"University of Salerno","ror":"https://ror.org/0192m2k53","country_code":"IT","type":"education","lineage":["https://openalex.org/I131729948"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Andrea De Lucia","raw_affiliation_strings":["Software Engineering (SeSa) Lab - Department of Computer Science, University of Salerno, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Software Engineering (SeSa) Lab - Department of Computer Science, University of Salerno, Italy","institution_ids":["https://openalex.org/I131729948"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5135980252","display_name":"Fabio Palomba","orcid":null},"institutions":[{"id":"https://openalex.org/I131729948","display_name":"University of Salerno","ror":"https://ror.org/0192m2k53","country_code":"IT","type":"education","lineage":["https://openalex.org/I131729948"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Fabio Palomba","raw_affiliation_strings":["Software Engineering (SeSa) Lab - Department of Computer Science, University of Salerno, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Software Engineering (SeSa) Lab - Department of Computer Science, University of Salerno, Italy","institution_ids":["https://openalex.org/I131729948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5107130597"],"corresponding_institution_ids":["https://openalex.org/I131729948"],"apc_list":{"value":3350,"currency":"USD","value_usd":3350},"apc_paid":{"value":3350,"currency":"USD","value_usd":3350},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.58508952,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"240","issue":null,"first_page":"112925","last_page":"112925"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.07419999688863754,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.07419999688863754,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.0406000018119812,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.033900000154972076,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.515999972820282},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.5094000101089478},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.4846999943256378},{"id":"https://openalex.org/keywords/quality-assurance","display_name":"Quality assurance","score":0.4844000041484833},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.474700003862381},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.46299999952316284},{"id":"https://openalex.org/keywords/integration-testing","display_name":"Integration testing","score":0.3741999864578247},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3659999966621399},{"id":"https://openalex.org/keywords/software-quality-assurance","display_name":"Software quality assurance","score":0.35830000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6492000222206116},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5401999950408936},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.515999972820282},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.5094000101089478},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.4846999943256378},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.4844000041484833},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47510001063346863},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.474700003862381},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.46299999952316284},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.43050000071525574},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4059000015258789},{"id":"https://openalex.org/C107683887","wikidata":"https://www.wikidata.org/wiki/Q782466","display_name":"Integration testing","level":3,"score":0.3741999864578247},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3659999966621399},{"id":"https://openalex.org/C2776969324","wikidata":"https://www.wikidata.org/wiki/Q613918","display_name":"Software quality assurance","level":5,"score":0.35830000042915344},{"id":"https://openalex.org/C182122060","wikidata":"https://www.wikidata.org/wiki/Q6752328","display_name":"Manual testing","level":5,"score":0.3102000057697296},{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.30970001220703125},{"id":"https://openalex.org/C131377759","wikidata":"https://www.wikidata.org/wiki/Q322514","display_name":"Acceptance testing","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C7435765","wikidata":"https://www.wikidata.org/wiki/Q7705776","display_name":"Test Management Approach","level":5,"score":0.288100004196167},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.2863999903202057},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.28529998660087585},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.28209999203681946},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C111524372","wikidata":"https://www.wikidata.org/wiki/Q7663718","display_name":"System integration testing","level":5,"score":0.2734000086784363},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.25839999318122864},{"id":"https://openalex.org/C169168650","wikidata":"https://www.wikidata.org/wiki/Q1675637","display_name":"Keyword-driven testing","level":5,"score":0.25429999828338623},{"id":"https://openalex.org/C63406617","wikidata":"https://www.wikidata.org/wiki/Q5266714","display_name":"Development testing","level":5,"score":0.2522999942302704}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.jss.2026.112925","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2026.112925","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.jss.2026.112925","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2026.112925","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320338450","display_name":"HORIZON EUROPE European Institute of Innovation and Technology","ror":null},{"id":"https://openalex.org/F5497039910","display_name":"Ministero dell'Istruzione e del Merito","ror":"https://ror.org/01ehyh486"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1479907160","https://openalex.org/W1560768931","https://openalex.org/W1978437060","https://openalex.org/W2023844160","https://openalex.org/W2322472622","https://openalex.org/W2324278080","https://openalex.org/W2518331092","https://openalex.org/W2903744551","https://openalex.org/W2994987245","https://openalex.org/W3004493192","https://openalex.org/W3007157104","https://openalex.org/W3016478929","https://openalex.org/W3045719304","https://openalex.org/W3046220160","https://openalex.org/W3087720408","https://openalex.org/W3091204980","https://openalex.org/W3105604018","https://openalex.org/W3122715378","https://openalex.org/W4281393087","https://openalex.org/W4284669642","https://openalex.org/W4308632711","https://openalex.org/W4312603874","https://openalex.org/W4312833928","https://openalex.org/W4387705798","https://openalex.org/W4391136507","https://openalex.org/W4394692327","https://openalex.org/W4399045945","https://openalex.org/W4400934506","https://openalex.org/W4401042136","https://openalex.org/W4401455160","https://openalex.org/W4403418001","https://openalex.org/W4412071673","https://openalex.org/W7117103063"],"related_works":[],"abstract_inverted_index":{"Machine":[0],"Learning":[1],"is":[2,31,70,193],"increasingly":[3],"embedded":[4],"in":[5,41,65,168,240],"critical":[6,165],"software":[7],"systems,":[8,29],"making":[9],"their":[10,86,184],"quality":[11],"assurance":[12],"a":[13,52,90,101,118,164,203,223,291],"matter":[14],"of":[15,62,104,120,230,236],"growing":[16],"concern.":[17],"While":[18],"the":[19,59,227],"research":[20,215],"community":[21],"has":[22],"proposed":[23,78],"several":[24],"techniques":[25,38,79,173,279],"for":[26,274],"testing":[27,47,64,111,150,238],"ML-enabled":[28],"there":[30],"limited":[32],"empirical":[33,54],"evidence":[34],"on":[35,137,153,260],"whether":[36,77],"these":[37],"are":[39,80,180],"adopted":[40],"practice":[42],"or":[43],"align":[44],"with":[45,206,226],"developers\u2019":[46],"workflows.":[48,209],"This":[49],"paper":[50],"presents":[51],"two-step":[53],"investigation":[55,235],"aimed":[56],"at":[57],"characterizing":[58],"current":[60],"landscape":[61],"ML":[63,237],"real-world":[66,241],"development.":[67,242],"Our":[68,130,187],"goal":[69],"to":[71,108,123,281,294],"understand":[72],"how":[73],"developers":[74,134],"approach":[75],"testing,":[76],"adopted,":[81],"and":[82,113,116,127,143,171,202,216,222,254,277,290],"what":[83],"barriers":[84],"hinder":[85],"implementation.":[87],"We":[88],"designed":[89],"mixed-method":[91,234],"study":[92,103],"that":[93,133,190,213],"triangulates":[94],"insights":[95,245],"from":[96,246],"two":[97],"complementary":[98],"sources:":[99],"(1)":[100],"mining":[102],"398":[105,247],"open-source":[106,248],"repositories":[107,249],"analyze":[109],"implemented":[110,147,182,266],"strategies":[112,139,262],"tool":[114],"usage;":[115],"(2)":[117],"survey":[119,188],"100":[121,255],"practitioners":[122],"capture":[124],"perceptions,":[125],"motivations,":[126],"practical":[128,196],"challenges.":[129],"findings":[131,211],"reveal":[132],"rely":[135,259],"heavily":[136],"foundational":[138,261],"like":[140,263],"Smoke":[141,264],"Testing":[142,177],"Rule-Based":[144],"Checking":[145],",":[146,158,178],"through":[148],"custom":[149,268],"logic":[151],"built":[152],"general-purpose":[154],"libraries":[155],"(e.g.,":[156],"PyTest":[157],"NumPy":[159],").":[160],"Conversely,":[161],"we":[162],"identified":[163],"adoption":[166,272],"gap":[167,192,273],"specialized":[169,275],"tools":[170,276],"advanced":[172,278],"such":[174],"as":[175],"Metamorphic":[176],"which":[179],"rarely":[181],"despite":[183],"academic":[185],"prominence.":[186],"indicates":[189],"this":[191],"driven":[194],"by":[195],"barriers,":[197],"including":[198],"high":[199],"integration":[200,283],"costs":[201],"poor":[204],"fit":[205],"existing":[207],"developer":[208],"These":[210],"suggest":[212],"future":[214],"tooling":[217],"must":[218],"prioritize":[219],"usability,":[220],"integration,":[221],"clearer":[224],"alignment":[225],"pragmatic":[228],"needs":[229],"developers.":[231],"\u2022":[232,243,257,270,285],"Large-scale":[233],"practices":[239],"Triangulated":[244],"(2,":[250],"018":[251],"test":[252],"files)":[253],"practitioners.":[256],"Practitioners":[258],"Testing,":[265],"via":[267],"solutions.":[269],"Critical":[271],"due":[280],"workflow":[282],"barriers.":[284],"Released":[286],"datasets,":[287],"analysis":[288],"scripts,":[289],"technical":[292],"report":[293],"enable":[295],"replication.":[296]},"counts_by_year":[],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2026-05-13T00:00:00"}