{"id":"https://openalex.org/W4383098200","doi":"https://doi.org/10.1109/cain58948.2023.00025","title":"Reproducibility Requires Consolidated Artifacts","display_name":"Reproducibility Requires Consolidated Artifacts","publication_year":2023,"publication_date":"2023-05-01","ids":{"openalex":"https://openalex.org/W4383098200","doi":"https://doi.org/10.1109/cain58948.2023.00025"},"language":"en","primary_location":{"id":"doi:10.1109/cain58948.2023.00025","doi":"https://doi.org/10.1109/cain58948.2023.00025","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/cain58948.2023.00025","pdf_url":null,"source":{"id":"https://openalex.org/S7407088863","display_name":"IEEE/ACM International Conference on AI Engineering \u2013 Software Engineering for AI (CAIN)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/ACM 2nd International Conference on AI Engineering \u2013 Software Engineering for AI (CAIN)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.12571","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087017450","display_name":"Iordanis Fostiropoulos","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Iordanis Fostiropoulos","raw_affiliation_strings":["University of Southern California, Los Angeles, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111003391","display_name":"Bowman Brown","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bowman Brown","raw_affiliation_strings":["University of Southern California, Los Angeles, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054494771","display_name":"Laurent Itti","orcid":"https://orcid.org/0000-0002-0168-2977","observed_orcids":["https://orcid.org/0000-0002-0168-2977"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Laurent Itti","raw_affiliation_strings":["University of Southern California, Los Angeles, CA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1174212"],"apc_list":null,"apc_paid":null,"fwci":0.7832,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.65680473,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"100","last_page":"101"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9355000257492065,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9355000257492065,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.03869999945163727,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.009600000455975533,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.8309999704360962},{"id":"https://openalex.org/keywords/machine-learning","display_name":"machine learning","score":0.7149999737739563},{"id":"https://openalex.org/keywords/hyperparameter-optimization","display_name":"hyperparameter optimization","score":0.6869999766349792},{"id":"https://openalex.org/keywords/unified-framework","display_name":"unified framework","score":0.5490000247955322},{"id":"https://openalex.org/keywords/replication-study","display_name":"replication study","score":0.7940000295639038},{"id":"https://openalex.org/keywords/source-code-repositories","display_name":"software repositories","score":0.8230000138282776},{"id":"https://openalex.org/keywords/replication-crisis","display_name":"replication crisis","score":0.6209999918937683}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.9298005700111389},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.8639316558837891},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7125459909439087},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.7028486728668213},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4352227747440338},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3519313931465149},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3367733359336853},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2530630826950073},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1221904456615448},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0690833330154419},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/cain58948.2023.00025","doi":"https://doi.org/10.1109/cain58948.2023.00025","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/cain58948.2023.00025","pdf_url":null,"source":{"id":"https://openalex.org/S7407088863","display_name":"IEEE/ACM International Conference on AI Engineering \u2013 Software Engineering for AI (CAIN)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/ACM 2nd International Conference on AI Engineering \u2013 Software Engineering for AI (CAIN)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2305.12571","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.12571","pdf_url":"https://arxiv.org/pdf/2305.12571","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2305.12571","doi":"https://doi.org/10.48550/arxiv.2305.12571","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.12571","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.12571","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.12571","pdf_url":"https://arxiv.org/pdf/2305.12571","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[{"display_name":"Systematic Review","id":"https://openalex.org/study-designs/systematic-review"}],"awards":[{"id":"https://openalex.org/G3879893252","display_name":null,"funder_award_id":"HR00112190134","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G486694964","display_name":null,"funder_award_id":"W911NF\u201020\u20102\u20100053","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"}],"funders":[{"id":"https://openalex.org/F4320306087","display_name":"Semiconductor Research Corporation","ror":"https://ror.org/047z4n946"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4383098200.pdf","grobid_xml":"https://content.openalex.org/works/W4383098200.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W2194775991","https://openalex.org/W2949676527","https://openalex.org/W2970283086","https://openalex.org/W3008058947","https://openalex.org/W3033015111","https://openalex.org/W3082154327","https://openalex.org/W3095209159","https://openalex.org/W3172701880"],"related_works":["https://openalex.org/W4385018165","https://openalex.org/W2941296231","https://openalex.org/W4409415146","https://openalex.org/W4322619436","https://openalex.org/W4407124756","https://openalex.org/W7168695879","https://openalex.org/W7115741582","https://openalex.org/W4387299663","https://openalex.org/W3200758773","https://openalex.org/W4389771368"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"is":[2],"facing":[3],"a":[4,8,30,73],"\u2018reproducibility":[5],"crisis\u2019":[6],"where":[7],"significant":[9],"number":[10],"of":[11,26,32,55,62],"works":[12],"report":[13],"failures":[14,28],"when":[15],"attempting":[16],"to":[17],"reproduce":[18],"previously":[19],"published":[20],"results.":[21],"We":[22,43,57],"evaluate":[23],"the":[24,60],"sources":[25],"reproducibility":[27],"using":[29],"meta-analysis":[31],"142":[33],"replication":[34],"studies":[35],"from":[36],"ReScience":[37],"C":[38],"and":[39,67],"204":[40],"code":[41],"repositories.":[42],"find":[44],"that":[45,69],"missing":[46],"experiment":[47],"details":[48],"such":[49],"as":[50],"hyperparameters":[51],"are":[52],"potential":[53],"causes":[54],"unreproducibility.":[56],"experimentally":[58],"show":[59],"bias":[61],"different":[63],"hyperparameter":[64],"selection":[65],"strategies":[66],"conclude":[68],"consolidated":[70],"artifacts":[71],"with":[72],"unified":[74],"framework":[75],"can":[76],"help":[77],"support":[78],"reproducibility.":[79]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2023-07-05T00:00:00"}