{"id":"https://openalex.org/W7161706840","doi":"https://doi.org/10.48550/arxiv.2605.16616","title":"MLReplicate: Benchmarking Autonomous Research Systems for Machine Learning Reproducibility","display_name":"MLReplicate: Benchmarking Autonomous Research Systems for Machine Learning Reproducibility","publication_year":2026,"publication_date":"2026-05-15","ids":{"openalex":"https://openalex.org/W7161706840","doi":"https://doi.org/10.48550/arxiv.2605.16616"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.16616","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16616","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.16616","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066758674","display_name":"Sasi Kiran Gaddipati","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gaddipati, Sasi Kiran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116166989","display_name":"Diyana Muhammed","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Muhammed, Diyana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116609038","display_name":"Farhana Keya","orcid":"https://orcid.org/0000-0002-3782-8069"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Keya, Farhana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028535577","display_name":"Gollam Rabby","orcid":"https://orcid.org/0000-0002-1212-0101"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rabby, Gollam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136490245","display_name":"S\u00f6ren Auer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Auer, S\u00f6ren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6905999779701233,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6905999779701233,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.1623000055551529,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.026799999177455902,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6991000175476074},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6247000098228455},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5778999924659729},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.429500013589859},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.41359999775886536},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.3912000060081482},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.36880001425743103}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7436000108718872},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6991000175476074},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6247000098228455},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6133999824523926},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5778999924659729},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5655999779701233},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.429500013589859},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.41359999775886536},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.3912000060081482},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.36880001425743103},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3650999963283539},{"id":"https://openalex.org/C9628104","wikidata":"https://www.wikidata.org/wiki/Q788009","display_name":"Autonomous system (mathematics)","level":2,"score":0.35519999265670776},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.33820000290870667},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.31709998846054077},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.2994999885559082},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2922999858856201},{"id":"https://openalex.org/C146047270","wikidata":"https://www.wikidata.org/wiki/Q469666","display_name":"Human\u2013machine system","level":2,"score":0.2696000039577484},{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.26260000467300415},{"id":"https://openalex.org/C34585555","wikidata":"https://www.wikidata.org/wiki/Q1368723","display_name":"Learning curve","level":2,"score":0.258899986743927}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.16616","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16616","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.16616","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.16616","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Autonomous":[0],"research":[1,35,60,205,223],"systems":[2,36,224],"capable":[3],"of":[4,107,118,160,213],"generating":[5],"complete":[6],"scientific":[7,227,242],"manuscripts":[8],"have":[9,18],"advanced":[10],"rapidly,":[11],"yet":[12],"robust":[13],"and":[14,55,71,94,104,152,158,225,229],"realistic":[15],"evaluation":[16,234],"frameworks":[17],"failed":[19,80],"to":[20,132,140],"keep":[21],"pace.":[22],"To":[23],"bridge":[24],"this":[25],"gap,":[26],"we":[27],"introduce":[28],"MLReplicate,":[29],"an":[30],"end-to-end":[31],"benchmark":[32,42],"evaluating":[33],"autonomous":[34,204,222],"on":[37],"machine":[38],"learning":[39],"reproducibility.":[40],"The":[41,111],"was":[43],"constructed":[44],"from":[45],"ICML":[46],"2025":[47],"outstanding":[48],"papers":[49],"reformulated":[50],"into":[51],"standardized":[52],"input":[53,198],"specifications":[54],"evaluated":[56],"across":[57,155],"6":[58],"state-of-the-art":[59],"systems:":[61],"AI":[62,64,69],"SCIENTIST-V1,":[63],"SCIENTIST-V2,":[65],"AGENT":[66],"LABORATORY,":[67],"CYCLERESEARCHER,":[68],"RESEARCHER,":[70],"TINY":[72],"SCIENTIST,":[73],"yielding":[74],"45":[75],"generated":[76],"manuscripts,":[77],"with":[78],"3":[79],"experiments.":[81],"Outputs":[82],"are":[83],"assessed":[84],"using":[85],"a":[86,194,217,231],"dual-protocol":[87],"approach":[88],"that":[89,172,203],"combines":[90],"automated":[91,112,141,162],"conference-style":[92,113],"review":[93,114,129],"structured":[95],"expert":[96],"human":[97,109,143,191],"evaluation,":[98,192],"while":[99],"tracking":[100],"computational":[101,177],"cost,":[102],"runtime,":[103],"the":[105,134,182,186,211],"amount":[106],"required":[108],"intervention.":[110],"accepted":[115,161],"10":[116],"out":[117],"37":[119],"valid":[120],"submissions.":[121],"An":[122],"additional":[123],"8":[124],"submissions":[125],"were":[126],"desk-rejected":[127],"before":[128],"for":[130,236],"failing":[131],"meet":[133],"minimum":[135],"page":[136],"threshold.":[137],"In":[138],"contrast":[139],"reviews,":[142],"reviewers":[144],"consistently":[145],"identified":[146],"methodological":[147],"flaws,":[148],"hallucinated":[149],"experimental":[150],"results,":[151],"reproducibility":[153],"failures":[154],"all":[156],"systems,":[157],"59%":[159],"reviews":[163],"contained":[164],"fabricated":[165],"or":[166],"unsupported":[167],"claims.":[168],"We":[169,200],"further":[170],"find":[171],"neither":[173],"token":[174],"budget":[175],"nor":[176],"cost":[178],"predicts":[179],"output":[180],"quality:":[181],"cheapest":[183],"system":[184,189],"outperforms":[185],"most":[187],"resource-intensive":[188],"in":[190,197],"despite":[193],"38-fold":[195],"difference":[196],"tokens.":[199],"thus":[201],"demonstrate":[202],"workflow":[206],"design":[207],"matters":[208],"more":[209],"than":[210],"scale":[212],"compute.":[214],"MLReplicate":[215],"exposes":[216],"substantial":[218],"gap":[219],"between":[220],"current":[221],"genuine":[226],"rigor,":[228],"establishes":[230],"practical,":[232],"extensible":[233],"framework":[235],"systematic":[237],"progress":[238],"toward":[239],"trustworthy":[240],"AI-driven":[241],"discovery.":[243]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-20T00:00:00"}