{"id":"https://openalex.org/W3023223120","doi":"https://doi.org/10.1145/3366424.3383417","title":"Expanding the Scope of Reproducibility Research Through Data Analysis Replications","display_name":"Expanding the Scope of Reproducibility Research Through Data Analysis Replications","publication_year":2020,"publication_date":"2020-04-20","ids":{"openalex":"https://openalex.org/W3023223120","doi":"https://doi.org/10.1145/3366424.3383417","mag":"3023223120"},"language":"en","primary_location":{"id":"doi:10.1145/3366424.3383417","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366424.3383417","pdf_url":null,"source":{"id":"https://openalex.org/S4306506651","display_name":"Companion Proceedings of the Web Conference 2020","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the Web Conference 2020","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3366424.3383417","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081429241","display_name":"Jake M. Hofman","orcid":"https://orcid.org/0000-0002-9364-9604"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jake M. Hofman","raw_affiliation_strings":["Microsoft Research New York, NY"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research New York, NY","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007746083","display_name":"Daniel G. Goldstein","orcid":"https://orcid.org/0000-0002-0970-5598"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel G. Goldstein","raw_affiliation_strings":["Microsoft Research New York, NY"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research New York, NY","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049055278","display_name":"Siddhartha Sen","orcid":"https://orcid.org/0000-0001-6501-6386"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siddhartha Sen","raw_affiliation_strings":["Microsoft Research New York, NY"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research New York, NY","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011235495","display_name":"Forough Poursabzi-Sandegh","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Forough Poursabzi-Sandegh","raw_affiliation_strings":["Microsoft Research New York, NY"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research New York, NY","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.5823,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.80448718,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"567","last_page":"571"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9782999753952026,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9782999753952026,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9775999784469604,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.964900016784668,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.6876327991485596},{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.6421923041343689},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.58537358045578},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4323107600212097},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.2302418053150177},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14218592643737793},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.06995242834091187}],"concepts":[{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.6876327991485596},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.6421923041343689},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.58537358045578},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4323107600212097},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2302418053150177},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14218592643737793},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.06995242834091187}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3366424.3383417","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366424.3383417","pdf_url":null,"source":{"id":"https://openalex.org/S4306506651","display_name":"Companion Proceedings of the Web Conference 2020","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the Web Conference 2020","raw_type":"proceedings-article"},{"id":"pmh:oai:RePEc:eee:jobhdp:v:164:y:2021:i:c:p:192-202","is_oa":false,"landing_page_url":"http://www.sciencedirect.com/science/article/pii/S0749597820304076","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:share.osf.io:1e8a1973-d4c2-4730-add3-94bd086ea108","is_oa":false,"landing_page_url":"https://osf.io/jue3b","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Project"}],"best_oa_location":{"id":"doi:10.1145/3366424.3383417","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366424.3383417","pdf_url":null,"source":{"id":"https://openalex.org/S4306506651","display_name":"Companion Proceedings of the Web Conference 2020","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the Web Conference 2020","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1897139626","https://openalex.org/W2118070418","https://openalex.org/W2161498332","https://openalex.org/W2166378641","https://openalex.org/W2322006099","https://openalex.org/W2414283705","https://openalex.org/W2584419678","https://openalex.org/W2729447387","https://openalex.org/W2779812635","https://openalex.org/W2789816182","https://openalex.org/W2806846062","https://openalex.org/W2890289215","https://openalex.org/W2952487143","https://openalex.org/W2973093011","https://openalex.org/W3121691535","https://openalex.org/W4243321652","https://openalex.org/W4243643633","https://openalex.org/W4253131989"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W3134060498","https://openalex.org/W2003643616"],"abstract_inverted_index":{"In":[0,66,195],"recent":[1],"years,":[2],"researchers":[3],"in":[4,122,134],"several":[5],"scientific":[6,41,166],"disciplines":[7],"have":[8,48],"become":[9],"concerned":[10],"with":[11],"published":[12,140],"studies":[13,168],"replicating":[14,31,135],"less":[15],"often":[16],"than":[17,154],"expected.":[18],"A":[19],"positive":[20],"side":[21],"effect":[22],"of":[23,39,90,108,139,150,165,202,224,228],"this":[24,67,232],"concern":[25],"is":[26,131],"an":[27],"increased":[28],"appreciation":[29],"for":[30,81,185],"other":[32],"researchers\u2019":[33],"work":[34],"as":[35],"a":[36,126,147,203,221,238],"vital":[37],"part":[38],"the":[40,51,116,136,161,187,191,211,225],"process.":[42],"To":[43],"date,":[44],"many":[45],"such":[46],"efforts":[47],"come":[49],"from":[50],"experimental":[52,72,85,119,141,155],"sciences,":[53],"where":[54],"replication":[55,73,86,120],"entails":[56],"running":[57],"new":[58,61],"experiments,":[59],"generating":[60],"data,":[62,110,112,114],"and":[63,115,152,163,190,219,234],"analyzing":[64],"it.":[65],"article,":[68],"we":[69,171,198,206],"emphasize":[70],"not":[71,94],"but":[74],"data":[75,101,137,178,216],"analysis":[76,179,217],"replication.":[77],"We":[78],"do":[79,93],"so":[80],"three":[82],"reasons.":[83],"First,":[84],"excludes":[87],"entire":[88],"classes":[89],"publications":[91],"that":[92,105],"run":[95],"experiments":[96],"or":[97],"even":[98],"collect":[99],"original":[100],"(for":[102],"example,":[103],"papers":[104],"make":[106],"use":[107],"economic":[109],"census":[111],"municipal":[113],"like).":[117],"Second,":[118],"may":[121],"some":[123],"cases":[124],"be":[125],"needlessly":[127],"high":[128],"bar:":[129],"there":[130],"great":[132],"value":[133],"analyses":[138],"work.":[142],"As":[143],"analytical":[144],"replications":[145],"require":[146],"lower":[148],"investment":[149],"time":[151],"money":[153],"replications,":[156,180,218],"their":[157],"adoption":[158],"should":[159],"expand":[160],"number":[162],"variety":[164],"reproducibility":[167],"undertaken.":[169],"Third,":[170],"propose":[172],"educating":[173],"undergraduate":[174],"students":[175,188,229],"to":[176,208,214,236],"perform":[177],"which":[181],"has":[182],"scalable":[183],"benefits":[184],"both":[186],"themselves":[189],"broader":[192],"research":[193],"community.":[194],"our":[196],"talk":[197],"will":[199],"provide":[200],"details":[201],"pilot":[204],"program":[205,233],"created":[207],"teach":[209],"undergraduates":[210],"skills":[212],"necessary":[213],"conduct":[215],"include":[220],"case":[222],"study":[223],"first":[226],"set":[227],"who":[230],"completed":[231],"attempted":[235],"replicate":[237],"widely-cited":[239],"social":[240],"science":[241],"paper":[242],"on":[243],"policing.":[244]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}