{"id":"https://openalex.org/W4409348647","doi":"https://doi.org/10.1609/aaai.v39i27.35093","title":"What Do Machine Learning Researchers Mean by \u201cReproducible\u201d?","display_name":"What Do Machine Learning Researchers Mean by \u201cReproducible\u201d?","alternate_titles":[],"publication_year":2025,"publication_date":"2025-04-11","ids":{"openalex":"https://openalex.org/W4409348647","doi":"https://doi.org/10.1609/aaai.v39i27.35093"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v39i27.35093","doi":"https://doi.org/10.1609/aaai.v39i27.35093","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i27.35093","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/35093/37248","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W4409348647"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/35093/37248","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068036546","display_name":"Edward Raff","orcid":"https://orcid.org/0000-0002-9900-1972","observed_orcids":["https://orcid.org/0000-0002-9900-1972"]},"institutions":[{"id":"https://openalex.org/I79272384","display_name":"University of Maryland, Baltimore County","ror":"https://ror.org/02qskvh78","country_code":"US","type":"education","lineage":["https://openalex.org/I79272384"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Edward Raff","raw_affiliation_strings":["Booz Allen Hamilton\nUniversity of Maryland, Baltimore County"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Booz Allen Hamilton\nUniversity of Maryland, Baltimore County","institution_ids":["https://openalex.org/I79272384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036782206","display_name":"Michel Benaroch","orcid":"https://orcid.org/0000-0002-8605-8814","observed_orcids":["https://orcid.org/0000-0002-8605-8814"]},"institutions":[{"id":"https://openalex.org/I70983195","display_name":"Syracuse University","ror":"https://ror.org/025r5qe02","country_code":"US","type":"education","lineage":["https://openalex.org/I70983195"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Michel Benaroch","raw_affiliation_strings":["Syracuse University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Syracuse University","institution_ids":["https://openalex.org/I70983195"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038811607","display_name":"Sagar Samtani","orcid":"https://orcid.org/0000-0002-4513-805X","observed_orcids":["https://orcid.org/0000-0002-4513-805X"]},"institutions":[{"id":"https://openalex.org/I592451","display_name":"Indiana University","ror":"https://ror.org/01kg8sb98","country_code":"US","type":"education","lineage":["https://openalex.org/I592451"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Sagar Samtani","raw_affiliation_strings":["Indiana University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indiana University","institution_ids":["https://openalex.org/I592451"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018573642","display_name":"Andrew Farris","orcid":"https://orcid.org/0000-0003-2023-9636","observed_orcids":["https://orcid.org/0000-0003-2023-9636"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Andrew L. Farris","raw_affiliation_strings":["Booz Allen Hamiltion"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Booz Allen Hamiltion","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":12.2514,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":{"value":0.98795181,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"39","issue":"27","first_page":"28671","last_page":"28683"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9262999892234802,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9262999892234802,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.006500000134110451,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.9570000171661377},{"id":"https://openalex.org/keywords/replication-crisis","display_name":"replication crisis","score":0.9879999756813049},{"id":"https://openalex.org/keywords/machine-learning-research","display_name":"machine learning research","score":0.8820000290870667},{"id":"https://openalex.org/keywords/research-reproducibility","display_name":"research reproducibility","score":0.7300000190734863}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.43881478905677795},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38269418478012085},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34879329800605774},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3377801179885864}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v39i27.35093","doi":"https://doi.org/10.1609/aaai.v39i27.35093","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i27.35093","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/35093/37248","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/35093","doi":null,"is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/35093","pdf_url":null,"source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:arXiv.org:2412.03854","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.03854","pdf_url":"https://arxiv.org/pdf/2412.03854","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2412.03854","doi":"https://doi.org/10.48550/arxiv.2412.03854","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2412.03854","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v39i27.35093","doi":"https://doi.org/10.1609/aaai.v39i27.35093","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i27.35093","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/35093/37248","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320310145","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95"},{"id":"https://openalex.org/F4320337253","display_name":"Instituto de Ciencias del Mar y Limnolog\u00eda, Universidad Nacional Aut\u00f3noma de M\u00e9xico","ror":"https://ror.org/040wdye98"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409348647.pdf","grobid_xml":"https://content.openalex.org/works/W4409348647.grobid-xml"},"referenced_works_count":135,"referenced_works":["https://openalex.org/W1508567255","https://openalex.org/W1565746575","https://openalex.org/W1617082848","https://openalex.org/W1631913055","https://openalex.org/W1833977909","https://openalex.org/W1991419023","https://openalex.org/W1992278063","https://openalex.org/W2081814119","https://openalex.org/W2132161525","https://openalex.org/W2137959503","https://openalex.org/W2141007997","https://openalex.org/W2141014056","https://openalex.org/W2142518823","https://openalex.org/W2150706391","https://openalex.org/W2155653793","https://openalex.org/W2159895197","https://openalex.org/W2167277498","https://openalex.org/W2189162242","https://openalex.org/W2292391400","https://openalex.org/W2404161646","https://openalex.org/W2490023472","https://openalex.org/W2608296545","https://openalex.org/W2612791305","https://openalex.org/W2708474221","https://openalex.org/W2765205444","https://openalex.org/W2770188460","https://openalex.org/W2772583926","https://openalex.org/W2785917027","https://openalex.org/W2788104422","https://openalex.org/W2788572835","https://openalex.org/W2798935874","https://openalex.org/W2886512263","https://openalex.org/W2888093235","https://openalex.org/W2890056718","https://openalex.org/W2890803903","https://openalex.org/W2891288076","https://openalex.org/W2896060389","https://openalex.org/W2896671475","https://openalex.org/W2898078244","https://openalex.org/W2898377472","https://openalex.org/W2904703100","https://openalex.org/W2908966008","https://openalex.org/W2912218663","https://openalex.org/W2913522468","https://openalex.org/W2915886991","https://openalex.org/W2941232686","https://openalex.org/W2946506331","https://openalex.org/W2950220847","https://openalex.org/W2953183526","https://openalex.org/W2957191877","https://openalex.org/W2963583956","https://openalex.org/W2970043916","https://openalex.org/W2971677963","https://openalex.org/W2972599464","https://openalex.org/W2998993395","https://openalex.org/W3010156474","https://openalex.org/W3011403448","https://openalex.org/W3011927872","https://openalex.org/W3026767020","https://openalex.org/W3034363135","https://openalex.org/W3035495837","https://openalex.org/W3088444111","https://openalex.org/W3108226083","https://openalex.org/W3109687052","https://openalex.org/W3122184684","https://openalex.org/W3128997905","https://openalex.org/W3130276836","https://openalex.org/W3135261467","https://openalex.org/W3137174954","https://openalex.org/W3138114698","https://openalex.org/W3146796711","https://openalex.org/W3160648246","https://openalex.org/W3173669456","https://openalex.org/W3174094293","https://openalex.org/W3174292387","https://openalex.org/W3174655995","https://openalex.org/W3174929685","https://openalex.org/W3177349994","https://openalex.org/W3183977027","https://openalex.org/W3198090350","https://openalex.org/W3199473923","https://openalex.org/W3200635595","https://openalex.org/W3210023107","https://openalex.org/W3211225893","https://openalex.org/W4200634785","https://openalex.org/W4210294742","https://openalex.org/W4220914596","https://openalex.org/W4221165557","https://openalex.org/W4226106048","https://openalex.org/W4235216760","https://openalex.org/W4240435199","https://openalex.org/W4280616270","https://openalex.org/W4282038170","https://openalex.org/W4283721498","https://openalex.org/W4284674986","https://openalex.org/W4284686595","https://openalex.org/W4284698080","https://openalex.org/W4284698859","https://openalex.org/W4284713500","https://openalex.org/W4285117098","https://openalex.org/W4285236184","https://openalex.org/W4286231567","https://openalex.org/W4287889441","https://openalex.org/W4289525739","https://openalex.org/W4312387258","https://openalex.org/W4313304974","https://openalex.org/W4319150656","https://openalex.org/W4376651356","https://openalex.org/W4376873982","https://openalex.org/W4382467693","https://openalex.org/W4385567251","https://openalex.org/W4385570577","https://openalex.org/W4385571947","https://openalex.org/W4385573073","https://openalex.org/W4385576721","https://openalex.org/W4385822840","https://openalex.org/W4388332846","https://openalex.org/W4388856981","https://openalex.org/W4396813955","https://openalex.org/W4399062457","https://openalex.org/W4400519494","https://openalex.org/W4400519535","https://openalex.org/W4400519911","https://openalex.org/W4402671955","https://openalex.org/W4415798757","https://openalex.org/W6922651736","https://openalex.org/W7027308578","https://openalex.org/W7033666630","https://openalex.org/W7103756286","https://openalex.org/W7133184161","https://openalex.org/W7133193041","https://openalex.org/W7133206787","https://openalex.org/W7133214952","https://openalex.org/W7133221448","https://openalex.org/W7133243906"],"related_works":["https://openalex.org/W4385018165","https://openalex.org/W4322619436","https://openalex.org/W4409415146","https://openalex.org/W4409574025","https://openalex.org/W7168695879","https://openalex.org/W7130532890","https://openalex.org/W7115741582","https://openalex.org/W4383098200","https://openalex.org/W6969433231","https://openalex.org/W4389771368"],"abstract_inverted_index":{"The":[0],"concern":[1],"that":[2,72,80],"Artificial":[3],"Intelligence":[4],"(AI)":[5],"and":[6],"Machine":[7],"Learning":[8],"(ML)":[9],"are":[10],"entering":[11],"a":[12],"\"reproducibility":[13],"crisis\"":[14],"has":[15],"spurred":[16],"significant":[17],"research":[18,61],"in":[19,89],"the":[20,42,49,60,97],"past":[21],"few":[22],"years.":[23],"Yet":[24],"with":[25],"each":[26,73],"paper,":[27],"it":[28],"is":[29],"often":[30],"unclear":[31],"what":[32],"someone":[33],"means":[34],"by":[35,48],"\"reproducibility\".":[36],"Our":[37],"work":[38],"attempts":[39],"to":[40,58,62,100],"clarify":[41],"scope":[43],"of":[44,74],"\"reproducibility\"":[45],"as":[46,85],"displayed":[47],"community":[50],"at":[51],"large.":[52],"In":[53,67],"doing":[54],"so,":[55],"we":[56,70],"propose":[57],"refine":[59],"eight":[63],"general":[64],"topic":[65],"areas.":[66],"this":[68],"light,":[69],"see":[71],"these":[75],"areas":[76],"contains":[77],"many":[78],"works":[79],"do":[81],"not":[82],"advertise":[83],"themselves":[84],"being":[86],"about":[87],"\"reproducibility\",":[88],"part":[90],"because":[91],"they":[92],"go":[93],"back":[94],"decades":[95],"before":[96],"matter":[98],"came":[99],"broader":[101],"attention.":[102]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4409348647","counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":5}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2025-04-11T00:00:00"}
