{"id":"https://openalex.org/W7204533236","doi":"https://doi.org/10.48550/arxiv.2608.27100","title":"Mutation Testing for Reproducibility Safeguards in Machine Learning Research Software: An Empirical Study","display_name":"Mutation Testing for Reproducibility Safeguards in Machine Learning Research Software: An Empirical Study","alternate_titles":[],"publication_year":2026,"publication_date":"2026-08-27","ids":{"openalex":"https://openalex.org/W7204533236","doi":"https://doi.org/10.48550/arxiv.2608.27100"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.27100","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27100","pdf_url":"https://arxiv.org/pdf/2608.27100","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7204533236"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.27100","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148426332","display_name":"Ilya Shulepov","orcid":"https://orcid.org/0009-0001-1348-9576","observed_orcids":["https://orcid.org/0009-0001-1348-9576"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shulepov, Ilya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5148426332"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5648999810218811,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5648999810218811,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.3855000138282776,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.025599999353289604,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mutation-testing","display_name":"mutation testing","score":0.8420000076293945},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.8629999756813049},{"id":"https://openalex.org/keywords/random-seed","display_name":"random seed","score":0.597000002861023},{"id":"https://openalex.org/keywords/dependency-pinning","display_name":"dependency pinning","score":0.9649999737739563},{"id":"https://openalex.org/keywords/data-splits","display_name":"data splits","score":0.734000027179718}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.7993000149726868},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6337000131607056},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.599399983882904},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.5848000049591064},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5677000284194946},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5559999942779541},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5059000253677368},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4887000024318695},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.47600001096725464},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4699999988079071},{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.43779999017715454},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.4228000044822693},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.3197000026702881},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3149000108242035},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.31299999356269836},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.29030001163482666},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.26100000739097595},{"id":"https://openalex.org/C3019813237","wikidata":"https://www.wikidata.org/wiki/Q65089264","display_name":"Model validation","level":2,"score":0.25519999861717224},{"id":"https://openalex.org/C2988343187","wikidata":"https://www.wikidata.org/wiki/Q1130645","display_name":"Open source software","level":3,"score":0.25060001015663147}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.27100","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27100","pdf_url":"https://arxiv.org/pdf/2608.27100","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.27100","doi":"https://doi.org/10.48550/arxiv.2608.27100","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.27100","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.27100","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27100","pdf_url":"https://arxiv.org/pdf/2608.27100","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204533236.pdf","grobid_xml":"https://content.openalex.org/works/W7204533236.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W2101700102","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2283438976","https://openalex.org/W2740215900","https://openalex.org/W2754517384","https://openalex.org/W2788572835","https://openalex.org/W2799640043","https://openalex.org/W2970043916","https://openalex.org/W3000442532","https://openalex.org/W3122184684","https://openalex.org/W3135261467","https://openalex.org/W3137174954","https://openalex.org/W3178469298","https://openalex.org/W3197433610","https://openalex.org/W3200467243","https://openalex.org/W3203026168","https://openalex.org/W4385576721"],"related_works":["https://openalex.org/W7204189871","https://openalex.org/W4407124756","https://openalex.org/W6991844361","https://openalex.org/W4410392447","https://openalex.org/W3196967454","https://openalex.org/W4394841360","https://openalex.org/W3113246766","https://openalex.org/W6892544630","https://openalex.org/W4386070929","https://openalex.org/W7159174410"],"abstract_inverted_index":{"Reproducibility":[0],"in":[1,57,162],"machine-learning":[2],"research":[3,38,47],"depends":[4],"on":[5],"experimental":[6],"choices":[7],"such":[8,30],"as":[9,187],"random":[10,75],"seeds,":[11],"dependency":[12,77],"versions,":[13],"data":[14,79],"partitioning,":[15],"and":[16,49,81,89],"evaluation":[17],"configuration.":[18],"Existing":[19],"repository":[20],"validation":[21,53,90,131,166],"workflows":[22,54,91,132,167],"may":[23],"execute":[24],"successfully":[25],"without":[26],"detecting":[27],"changes":[28,176],"to":[29,45,117,140,191],"choices.":[31,199],"We":[32,60],"study":[33,65],"this":[34,163],"problem":[35],"using":[36,71],"MLReproMutate,":[37],"software":[39],"that":[40,152],"applies":[41],"controlled,":[42],"reproducibility-relevant":[43,175],"mutations":[44,127],"ML":[46],"repositories":[48,155],"evaluates":[50],"them":[51],"against":[52],"already":[55],"present":[56],"those":[58],"repositories.":[59],"conducted":[61],"an":[62,141],"outcome-blind":[63],"empirical":[64],"of":[66,105,135,145],"39":[67,106],"frozen":[68],"repository-operator":[69],"cases":[70],"four":[72],"mutation":[73,87,95,185],"classes:":[74],"seed,":[76],"pin,":[78],"split,":[80],"cross-validation":[82],"fold":[83],"count.":[84],"Repository":[85],"revisions,":[86],"candidates,":[88],"were":[92,97],"fixed":[93],"before":[94],"outcomes":[96,102],"observed.":[98],"Primary":[99],"execution":[100],"yielded":[101],"for":[103],"13":[104],"cases;":[107],"a":[108,188],"bounded":[109],"restoration":[110],"procedure":[111],"increased":[112],"the":[113,153,172,179],"combined":[114],"evaluable":[115],"set":[116],"24.":[118],"After":[119],"excluding":[120],"one":[121],"confirmed-equivalent":[122],"mutation,":[123],"23":[124,137],"confirmed":[125],"non-equivalent":[126],"remained.":[128],"The":[129,181],"selected":[130],"detected":[133],"2":[134],"these":[136],"mutations,":[138],"corresponding":[139,154],"observed":[142],"detection":[143],"proportion":[144],"8.7%.":[146],"These":[147],"results":[148],"do":[149],"not":[150,170],"imply":[151],"are":[156],"irreproducible.":[157],"Rather,":[158],"they":[159],"show":[160],"that,":[161],"sample,":[164],"existing":[165],"often":[168],"did":[169],"detect":[171],"particular":[173],"controlled":[174],"introduced":[177],"by":[178],"study.":[180],"findings":[182],"motivate":[183],"reproducibility-oriented":[184],"testing":[186],"complementary":[189],"way":[190],"assess":[192],"whether":[193],"research-software":[194],"safeguards":[195],"constrain":[196],"experimentally":[197],"important":[198]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7204533236","counts_by_year":[],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2026-08-29T00:00:00"}
