{"id":"https://openalex.org/W7130640968","doi":"https://doi.org/10.1007/s10664-025-10797-w","title":"An audit of machine learning experiments on software defect prediction","display_name":"An audit of machine learning experiments on software defect prediction","publication_year":2026,"publication_date":"2026-02-20","ids":{"openalex":"https://openalex.org/W7130640968","doi":"https://doi.org/10.1007/s10664-025-10797-w"},"language":"en","primary_location":{"id":"doi:10.1007/s10664-025-10797-w","doi":"https://doi.org/10.1007/s10664-025-10797-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-025-10797-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10797-w.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10797-w.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036425614","display_name":"Giuseppe Destefanis","orcid":"https://orcid.org/0000-0003-3982-6355","observed_orcids":["https://orcid.org/0000-0003-3982-6355"]},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Giuseppe Destefanis","raw_affiliation_strings":["Department of Computer Science, University College London, London, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University College London, London, UK","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103128438","display_name":"Leila Yousefi","orcid":"https://orcid.org/0000-0003-1952-0674","observed_orcids":["https://orcid.org/0000-0003-1952-0674"]},"institutions":[{"id":"https://openalex.org/I59433898","display_name":"Brunel University of London","ror":"https://ror.org/00dn4t376","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I59433898"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Leila Yousefi","raw_affiliation_strings":["Department of Computer Science, Brunel University London, West London, UB8 3PH, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brunel University London, West London, UB8 3PH, UK","institution_ids":["https://openalex.org/I59433898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027684680","display_name":"Martin Shepperd","orcid":"https://orcid.org/0000-0003-1874-6145","observed_orcids":["https://orcid.org/0000-0003-1874-6145"]},"institutions":[{"id":"https://openalex.org/I59433898","display_name":"Brunel University of London","ror":"https://ror.org/00dn4t376","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I59433898"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Martin Shepperd","raw_affiliation_strings":["Department of Computer Science, Brunel University London, West London, UB8 3PH, UK"],"raw_orcid":"https://orcid.org/0000-0003-1874-6145","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brunel University London, West London, UB8 3PH, UK","institution_ids":["https://openalex.org/I59433898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123940051","display_name":"Allan Tucker","orcid":"https://orcid.org/0000-0001-5105-3506","observed_orcids":["https://orcid.org/0000-0001-5105-3506"]},"institutions":[{"id":"https://openalex.org/I59433898","display_name":"Brunel University of London","ror":"https://ror.org/00dn4t376","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I59433898"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Allan Tucker","raw_affiliation_strings":["Department of Computer Science, Brunel University London, West London, UB8 3PH, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brunel University London, West London, UB8 3PH, UK","institution_ids":["https://openalex.org/I59433898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042037650","display_name":"Stephen Swift","orcid":"https://orcid.org/0000-0001-8918-3365","observed_orcids":["https://orcid.org/0000-0001-8918-3365"]},"institutions":[{"id":"https://openalex.org/I59433898","display_name":"Brunel University of London","ror":"https://ror.org/00dn4t376","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I59433898"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Stephen Swift","raw_affiliation_strings":["Department of Computer Science, Brunel University London, West London, UB8 3PH, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brunel University London, West London, UB8 3PH, UK","institution_ids":["https://openalex.org/I59433898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083706258","display_name":"Steve Counsell","orcid":"https://orcid.org/0000-0002-2939-8919","observed_orcids":["https://orcid.org/0000-0002-2939-8919"]},"institutions":[{"id":"https://openalex.org/I59433898","display_name":"Brunel University of London","ror":"https://ror.org/00dn4t376","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I59433898"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Steve Counsell","raw_affiliation_strings":["Department of Computer Science, Brunel University London, West London, UB8 3PH, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brunel University London, West London, UB8 3PH, UK","institution_ids":["https://openalex.org/I59433898"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077607362","display_name":"Mahir Arzoky","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I59433898","display_name":"Brunel University of London","ror":"https://ror.org/00dn4t376","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I59433898"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mahir Arzoky","raw_affiliation_strings":["Department of Computer Science, Brunel University London, West London, UB8 3PH, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brunel University London, West London, UB8 3PH, UK","institution_ids":["https://openalex.org/I59433898"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5027684680"],"corresponding_institution_ids":["https://openalex.org/I59433898"],"apc_list":{"value":2290,"currency":"USD","value_usd":2290},"apc_paid":{"value":2290,"currency":"USD","value_usd":2290},"fwci":32.6282,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.9954527,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"31","issue":"4","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.002899999963119626,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-defect-prediction","display_name":"software defect prediction","score":1.0},{"id":"https://openalex.org/keywords/experimental-design","display_name":"experimental design","score":0.5529999732971191},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.890999972820282},{"id":"https://openalex.org/keywords/out-of-sample-validation","display_name":"out-of-sample validation","score":0.8149999976158142},{"id":"https://openalex.org/keywords/statistical-inference","display_name":"statistical inference","score":0.7910000085830688},{"id":"https://openalex.org/keywords/performance-indicators","display_name":"performance indicators","score":0.8059999942779541},{"id":"https://openalex.org/keywords/paper-mills","display_name":"paper mills","score":0.5289999842643738},{"id":"https://openalex.org/keywords/research-integrity","display_name":"research integrity","score":0.375}],"concepts":[{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.7656000256538391},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7264999747276306},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6539000272750854},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.5788000226020813},{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.5501999855041504},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5347999930381775},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.5069000124931335},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.482699990272522},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.44040000438690186},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.38670000433921814},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.367000013589859},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36660000681877136},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.36169999837875366},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.349700003862381},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.32420000433921814},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.31290000677108765},{"id":"https://openalex.org/C84525736","wikidata":"https://www.wikidata.org/wiki/Q831366","display_name":"Decision tree","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C34585555","wikidata":"https://www.wikidata.org/wiki/Q1368723","display_name":"Learning curve","level":2,"score":0.2888000011444092},{"id":"https://openalex.org/C110083411","wikidata":"https://www.wikidata.org/wiki/Q1744628","display_name":"Statistical classification","level":2,"score":0.2881999909877777},{"id":"https://openalex.org/C89567784","wikidata":"https://www.wikidata.org/wiki/Q7554325","display_name":"Software measurement","level":5,"score":0.2743000090122223},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.26980000734329224},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.26589998602867126},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.263700008392334}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1007/s10664-025-10797-w","doi":"https://doi.org/10.1007/s10664-025-10797-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-025-10797-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10797-w.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2601.18477","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2601.18477","pdf_url":"https://arxiv.org/pdf/2601.18477","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:bura.brunel.ac.uk:2438/33014","doi":null,"is_oa":true,"landing_page_url":"https://orcid.org/0000-0003-1874-6145","pdf_url":null,"source":{"id":"https://openalex.org/S4306401473","display_name":"Brunel University Research Archive (BURA) (Brunel University London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I59433898","host_organization_name":"Brunel University of London","host_organization_lineage":["https://openalex.org/I59433898"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2601.18477","doi":"https://doi.org/10.48550/arxiv.2601.18477","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.18477","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1007/s10664-025-10797-w","doi":"https://doi.org/10.1007/s10664-025-10797-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-025-10797-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10797-w.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-a","jufo-3","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7130640968.pdf","grobid_xml":"https://content.openalex.org/works/W7130640968.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W1590984175","https://openalex.org/W1897139626","https://openalex.org/W1910875068","https://openalex.org/W2023399666","https://openalex.org/W2029445248","https://openalex.org/W2030031047","https://openalex.org/W2040032582","https://openalex.org/W2063876764","https://openalex.org/W2089135290","https://openalex.org/W2089745446","https://openalex.org/W2090625011","https://openalex.org/W2096942889","https://openalex.org/W2107432340","https://openalex.org/W2110065044","https://openalex.org/W2118283821","https://openalex.org/W2135445066","https://openalex.org/W2144981148","https://openalex.org/W2163851162","https://openalex.org/W2408017125","https://openalex.org/W2570760970","https://openalex.org/W2606460416","https://openalex.org/W2800788706","https://openalex.org/W2885134687","https://openalex.org/W2906829543","https://openalex.org/W2942137712","https://openalex.org/W2972590742","https://openalex.org/W2999309192","https://openalex.org/W3001554335","https://openalex.org/W3006568433","https://openalex.org/W3009290003","https://openalex.org/W3030391949","https://openalex.org/W3091970108","https://openalex.org/W3109355298","https://openalex.org/W3113246766","https://openalex.org/W3122041120","https://openalex.org/W3133820354","https://openalex.org/W3170490302","https://openalex.org/W3172444956","https://openalex.org/W3173291597","https://openalex.org/W3182844710","https://openalex.org/W3211225893","https://openalex.org/W4223430324","https://openalex.org/W4229056796","https://openalex.org/W4298630190","https://openalex.org/W4310186689","https://openalex.org/W4313480230","https://openalex.org/W4319302553","https://openalex.org/W4385576721","https://openalex.org/W4386070929","https://openalex.org/W4386175986","https://openalex.org/W4388843875","https://openalex.org/W4390674594","https://openalex.org/W4400721937","https://openalex.org/W4412934489"],"related_works":["https://openalex.org/W4313480230","https://openalex.org/W7161496607","https://openalex.org/W2972590742","https://openalex.org/W4410267360","https://openalex.org/W4327715141","https://openalex.org/W7223263547","https://openalex.org/W4389041535","https://openalex.org/W4405204881","https://openalex.org/W7165044948","https://openalex.org/W2791812915"],"abstract_inverted_index":{"Machine":[0],"learning":[1,68],"algorithms":[2],"are":[3,21,195],"increasingly":[4],"being":[5,258],"proposed":[6,156],"to":[7,76,230,270,290,336],"solve":[8],"the":[9,22,31,78,85,99,112,119,131,154,212,223,301],"problem":[10],"of":[11,25,33,81,87,94,121,123,151,173,207,214,225,234,238,246,263,281],"predicting":[12],"defect-prone":[13],"software":[14,47,71],"components.":[15],"In":[16,137,261],"this":[17],"literature,":[18],"computational":[19,294],"experiments":[20,51,166,265,295],"primary":[23],"means":[24],"evaluating":[26],"and":[27,30,41,58,69,83,108,118,134,159,189,222,284,292],"comparing":[28],"learners":[29,215],"credibility":[32],"findings":[34],"depends":[35],"critically":[36],"on":[37,104],"their":[38,54,105],"experimental":[39,55,106],"design":[40,107],"reporting.":[42],"This":[43,145],"paper":[44,257],"audits":[45],"recent":[46],"defect":[48],"prediction":[49],"(SDP)":[50],"by":[52,148,157],"assessing":[53],"design,":[56],"analysis":[57,109,133],"reporting":[59,293],"practices":[60],"against":[61],"widely":[62],"accepted":[63],"norms":[64],"from":[65,98,220,228,267],"statistics,":[66],"machine":[67],"empirical":[70],"engineering.":[72],"Our":[73,161],"aim":[74],"is":[75],"characterise":[77],"current":[79],"state":[80],"practice":[82],"evaluate":[84],"reproducibility":[86,152],"published":[88,97],"findings.":[89],"We":[90,198,242,276],"undertook":[91],"an":[92,149],"audit":[93,314],"relevant":[95],"studies":[96,302],"SCOPUS":[100],"database":[101],"(2019-2023)":[102],"focusing":[103],"choices":[110],"e.g.,":[111,128],"outcome":[113],"variables":[114],"such":[115,306],"as":[116,317],"F-measure":[117],"type":[120],"out":[122],"sample":[124],"(OOS)":[125],"validation":[126],"regime,":[127],"cross-validation,":[129],"plus":[130],"statistical":[132,240],"inference":[135],"mechanisms.":[136],"all,":[138],"we":[139,177,321],"evaluated":[140,219],"nine":[141],"different":[142],"study":[143],"issues.":[144],"was":[146],"complemented":[147],"assessment":[150],"using":[153],"instrument":[155],"Gonz\u00e1lez-Barahona":[158],"Robles.":[160],"search":[162],"located":[163],"approximately":[164],"1,585":[165],"in":[167,202],"SDP":[168],"(2019-2023),":[169],"a":[170,244,318],"substantial":[171],"body":[172],"work.":[174],"From":[175],"this,":[176],"randomly":[178],"sampled":[179],"101":[180,251],"(":[181],"$$":[182],"\\approx":[183],"6.4\\%$$":[184],")":[185],"papers,":[186],"61":[187],"journal":[188],"40":[190],"conference":[191],"papers.":[192],"Almost":[193],"50%":[194],"behind":[196],"\u2018paywalls\u2019.":[197],"found":[199,278],"considerable":[200,323],"divergence":[201],"research":[203,319],"practice.":[204],"The":[205],"number":[206,213,224],"datasets":[208],"used":[209],"ranged":[210,266],"1-365,":[211],"or":[216],"learner":[217],"variants":[218],"1-34":[221],"performance":[226],"metrics":[227],"1":[229],"9.":[231],"Approximately":[232],"45%":[233],"papers":[235,252],"made":[236],"use":[237],"formal":[239],"inference.":[241],"detected":[243],"total":[245],"427":[247],"issues":[248],"distributed":[249],"across":[250],"(median=4)":[253],"with":[254],"only":[255],"one":[256],"entirely":[259],"issue-free.":[260],"terms":[262],"reproducibility,":[264],"near":[268],"perfect":[269],"lacking":[271],"almost":[272,299],"all":[273],"required":[274],"information.":[275],"also":[277],"two":[279],"examples":[280],"tortured":[282],"phrases":[283],"potential":[285],"\u201cpaper":[286],"mill\u201d":[287],"activity.":[288],"Approaches":[289],"designing":[291],"varied":[296],"greatly,":[297],"but":[298],"half":[300],"provided":[303],"insufficient":[304],"information":[305],"that":[307,316],"reproduction":[308],"would":[309],"be":[310,331],"challenging.":[311],"Overall,":[312],"our":[313],"suggests":[315],"community,":[320],"have":[322],"scope":[324],"for":[325],"improvement.":[326],"Fortunately,":[327],"many":[328],"improvements":[329],"should":[330],"neither":[332],"difficult":[333],"nor":[334],"costly":[335],"achieve.":[337]},"counts_by_year":[{"year":2026,"cited_by_count":9}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-02-20T00:00:00"}