{"id":"https://openalex.org/W3044177754","doi":"https://doi.org/10.1145/3360311","title":"Threats of a replication crisis in empirical computer science","display_name":"Threats of a replication crisis in empirical computer science","publication_year":2020,"publication_date":"2020-07-22","ids":{"openalex":"https://openalex.org/W3044177754","doi":"https://doi.org/10.1145/3360311","mag":"3044177754"},"language":"en","primary_location":{"id":"doi:10.1145/3360311","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3360311","pdf_url":null,"source":{"id":"https://openalex.org/S103482838","display_name":"Communications of the ACM","issn_l":"0001-0782","issn":["0001-0782","1557-7317"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications of the ACM","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-171501","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080181433","display_name":"Andy Cockburn","orcid":"https://orcid.org/0000-0003-1074-4268"},"institutions":[{"id":"https://openalex.org/I185492890","display_name":"University of Canterbury","ror":"https://ror.org/03y7q9t39","country_code":"NZ","type":"education","lineage":["https://openalex.org/I185492890"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Andy Cockburn","raw_affiliation_strings":["University of Cantebury, Christchurch, New Zealand"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cantebury, Christchurch, New Zealand","institution_ids":["https://openalex.org/I185492890"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032305401","display_name":"Pierre Dragicevic","orcid":"https://orcid.org/0000-0002-1854-5899"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Pierre Dragicevic","raw_affiliation_strings":["Inria, Orsay, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inria, Orsay, France","institution_ids":["https://openalex.org/I1326498283"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087889244","display_name":"Lonni Besan\u00e7on","orcid":"https://orcid.org/0000-0002-7207-1276"},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Lonni Besan\u00e7on","raw_affiliation_strings":["Link\u00f6ping University, Link\u00f6ping, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Link\u00f6ping University, Link\u00f6ping, Sweden","institution_ids":["https://openalex.org/I102134673"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081930058","display_name":"Carl Gutwin","orcid":"https://orcid.org/0000-0003-4470-9495"},"institutions":[{"id":"https://openalex.org/I32625721","display_name":"University of Saskatchewan","ror":"https://ror.org/010x8gc63","country_code":"CA","type":"education","lineage":["https://openalex.org/I32625721"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Carl Gutwin","raw_affiliation_strings":["University of Saskatchewan, Canada","U of S - University of Saskatchewan [Saskatoon, Canada] (Saskatoon, Saskatchewan Canada - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Saskatchewan, Canada","institution_ids":["https://openalex.org/I32625721"]},{"raw_affiliation_string":"U of S - University of Saskatchewan [Saskatoon, Canada] (Saskatoon, Saskatchewan Canada - Canada)","institution_ids":["https://openalex.org/I32625721"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":40.9575,"has_fulltext":false,"cited_by_count":153,"citation_normalized_percentile":{"value":0.99658599,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"63","issue":"8","first_page":"70","last_page":"79"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9258000254631042,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.9002583026885986},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.47977906465530396},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4306848645210266},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.26883476972579956},{"id":"https://openalex.org/keywords/virology","display_name":"Virology","score":0.13050490617752075},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.10985895991325378},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.0685499906539917}],"concepts":[{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.9002583026885986},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47977906465530396},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4306848645210266},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.26883476972579956},{"id":"https://openalex.org/C159047783","wikidata":"https://www.wikidata.org/wiki/Q7215","display_name":"Virology","level":1,"score":0.13050490617752075},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.10985895991325378},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0685499906539917}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3360311","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3360311","pdf_url":null,"source":{"id":"https://openalex.org/S103482838","display_name":"Communications of the ACM","issn_l":"0001-0782","issn":["0001-0782","1557-7317"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications of the ACM","raw_type":"journal-article"},{"id":"pmh:oai:DiVA.org:liu-171501","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-171501","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article, review/survey"},{"id":"pmh:oai:HAL:hal-02907143v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-02907143","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Communications of the ACM, 2020, 63 (8), pp.70-79. ⟨10.1145/3360311⟩","raw_type":"Journal articles"}],"best_oa_location":{"id":"pmh:oai:DiVA.org:liu-171501","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-171501","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article, review/survey"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W1500693574","https://openalex.org/W1944584781","https://openalex.org/W1993266153","https://openalex.org/W1996977712","https://openalex.org/W1997809363","https://openalex.org/W2006546769","https://openalex.org/W2032593675","https://openalex.org/W2039399957","https://openalex.org/W2042983463","https://openalex.org/W2054690084","https://openalex.org/W2062908954","https://openalex.org/W2075585362","https://openalex.org/W2081569317","https://openalex.org/W2100267603","https://openalex.org/W2118070418","https://openalex.org/W2127875809","https://openalex.org/W2136634361","https://openalex.org/W2144981148","https://openalex.org/W2157698673","https://openalex.org/W2161498332","https://openalex.org/W2275632435","https://openalex.org/W2283438976","https://openalex.org/W2406493898","https://openalex.org/W2538248335","https://openalex.org/W2589849966","https://openalex.org/W2610297457","https://openalex.org/W2758081422","https://openalex.org/W2788572835","https://openalex.org/W2793021873","https://openalex.org/W2804443912","https://openalex.org/W2946096838","https://openalex.org/W2949268160","https://openalex.org/W3124625789","https://openalex.org/W3124690569","https://openalex.org/W4237895524","https://openalex.org/W4256541692","https://openalex.org/W6741052740"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W4205713785","https://openalex.org/W2001405890","https://openalex.org/W3016766501","https://openalex.org/W2901033488","https://openalex.org/W2382290278","https://openalex.org/W2478288626"],"abstract_inverted_index":{"Research":[0],"replication":[1],"only":[2],"works":[3],"if":[4],"there":[5],"is":[6],"confidence":[7],"built":[8],"into":[9],"the":[10],"results.":[11]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":24},{"year":2024,"cited_by_count":32},{"year":2023,"cited_by_count":32},{"year":2022,"cited_by_count":32},{"year":2021,"cited_by_count":23},{"year":2020,"cited_by_count":3}],"updated_date":"2026-06-13T07:54:00.901334","created_date":"2025-10-10T00:00:00"}