{"id":"https://openalex.org/W2891288076","doi":"https://doi.org/10.1162/coli_a_00330","title":"Reproducibility in Computational Linguistics: Are We Willing to Share?","display_name":"Reproducibility in Computational Linguistics: Are We Willing to Share?","publication_year":2018,"publication_date":"2018-09-18","ids":{"openalex":"https://openalex.org/W2891288076","doi":"https://doi.org/10.1162/coli_a_00330","mag":"2891288076"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00330","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00330","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/coli_a_00330","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/coli_a_00330","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075698724","display_name":"Martijn Wieling","orcid":"https://orcid.org/0000-0003-0434-1526"},"institutions":[{"id":"https://openalex.org/I1334415907","display_name":"University Medical Center Groningen","ror":"https://ror.org/03cv38k47","country_code":"NL","type":"healthcare","lineage":["https://openalex.org/I1334415907"]},{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Martijn Wieling","raw_affiliation_strings":["University of Groningen, Center for Language and Cognition, Groningen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, Center for Language and Cognition, Groningen","institution_ids":["https://openalex.org/I169381384","https://openalex.org/I1334415907"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075365064","display_name":"Josine Rawee","orcid":"https://orcid.org/0000-0002-7603-9417"},"institutions":[{"id":"https://openalex.org/I1334415907","display_name":"University Medical Center Groningen","ror":"https://ror.org/03cv38k47","country_code":"NL","type":"healthcare","lineage":["https://openalex.org/I1334415907"]},{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Josine Rawee","raw_affiliation_strings":["Master\u2019s student, University of Groningen, Center for Language and Cognition, Groningen","Master's student, University of Groningen, Center for Language and Cognition, Groningen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Master\u2019s student, University of Groningen, Center for Language and Cognition, Groningen","institution_ids":["https://openalex.org/I169381384","https://openalex.org/I1334415907"]},{"raw_affiliation_string":"Master's student, University of Groningen, Center for Language and Cognition, Groningen","institution_ids":["https://openalex.org/I169381384","https://openalex.org/I1334415907"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047337772","display_name":"Gertjan van Noord","orcid":"https://orcid.org/0000-0001-5564-6341"},"institutions":[{"id":"https://openalex.org/I1334415907","display_name":"University Medical Center Groningen","ror":"https://ror.org/03cv38k47","country_code":"NL","type":"healthcare","lineage":["https://openalex.org/I1334415907"]},{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Gertjan van Noord","raw_affiliation_strings":["University of Groningen, Center for Language and Cognition, Groningen"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, Center for Language and Cognition, Groningen","institution_ids":["https://openalex.org/I169381384","https://openalex.org/I1334415907"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5047337772","https://openalex.org/A5075365064","https://openalex.org/A5075698724"],"corresponding_institution_ids":["https://openalex.org/I1334415907","https://openalex.org/I169381384"],"apc_list":null,"apc_paid":null,"fwci":4.0649,"has_fulltext":true,"cited_by_count":66,"citation_normalized_percentile":{"value":0.95066391,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":"44","issue":"4","first_page":"641","last_page":"649"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6525093913078308},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5968086123466492},{"id":"https://openalex.org/keywords/computational-linguistics","display_name":"Computational linguistics","score":0.581974446773529},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.5363138318061829},{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.5086801648139954},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4900249242782593},{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.46842917799949646},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4130893349647522},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.4109894335269928},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3665352761745453},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.3434877395629883},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.2988913059234619},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21887457370758057},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.19532564282417297},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1915486752986908},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12946969270706177},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.11936306953430176},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.1173970103263855},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.09863844513893127},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.09168455004692078}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6525093913078308},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5968086123466492},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.581974446773529},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.5363138318061829},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.5086801648139954},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4900249242782593},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.46842917799949646},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4130893349647522},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.4109894335269928},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3665352761745453},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3434877395629883},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2988913059234619},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21887457370758057},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.19532564282417297},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1915486752986908},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12946969270706177},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.11936306953430176},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.1173970103263855},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.09863844513893127},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.09168455004692078},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1162/coli_a_00330","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00330","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/coli_a_00330","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:pure.rug.nl:publications/da488788-f422-4139-a1b8-1661483d3a47","is_oa":true,"landing_page_url":"https://research.rug.nl/en/publications/da488788-f422-4139-a1b8-1661483d3a47","pdf_url":"https://pure.rug.nl/ws/files/75405323/coli_a_00330.pdf","source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Wieling, M, Rawee, J & van Noord, G 2018, 'Reproducibility in Computational Linguistics : Are We Willing to Share?', Computational Linguistics, vol. 44, no. 4, pp. 641-649. https://doi.org/10.1162/coli_a_00330","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:doaj.org/article:155f8b6efe174058b23fd84833a03cc7","is_oa":false,"landing_page_url":"https://doaj.org/article/155f8b6efe174058b23fd84833a03cc7","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 44, Iss 4, Pp 641-649 (2018)","raw_type":"article"},{"id":"pmh:oai:pure.rug.nl:openaire_cris_publications/da488788-f422-4139-a1b8-1661483d3a47","is_oa":true,"landing_page_url":"https://hdl.handle.net/11370/da488788-f422-4139-a1b8-1661483d3a47","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Wieling, M, Rawee, J & van Noord, G 2018, 'Reproducibility in Computational Linguistics : Are We Willing to Share?', Computational Linguistics, vol. 44, no. 4, pp. 641-649. https://doi.org/10.1162/coli_a_00330","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00330","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00330","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/coli_a_00330","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7900000214576721,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2891288076.pdf","grobid_xml":"https://content.openalex.org/works/W2891288076.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1975004953","https://openalex.org/W1985754308","https://openalex.org/W2092654472","https://openalex.org/W2103883986","https://openalex.org/W2130126144","https://openalex.org/W2151752537","https://openalex.org/W2156036736","https://openalex.org/W2161002933","https://openalex.org/W2251964473","https://openalex.org/W2507762574","https://openalex.org/W2511646563","https://openalex.org/W2515464445","https://openalex.org/W2588066183","https://openalex.org/W2740834240","https://openalex.org/W2787858429","https://openalex.org/W2963687836","https://openalex.org/W2963949413","https://openalex.org/W3121854931","https://openalex.org/W4229954146"],"related_works":["https://openalex.org/W2344367508","https://openalex.org/W2904997879","https://openalex.org/W3081644756","https://openalex.org/W2097696338","https://openalex.org/W20188161","https://openalex.org/W4318539024","https://openalex.org/W2969773072","https://openalex.org/W2909969119","https://openalex.org/W4389600167","https://openalex.org/W4285169523"],"abstract_inverted_index":{"This":[0],"study":[1],"focuses":[2],"on":[3],"an":[4],"essential":[5],"precondition":[6],"for":[7,151,207,217,224],"reproducibility":[8,202],"in":[9,32,41,112,119,183],"computational":[10,42,184],"linguistics:":[11],"the":[12,62,113,117,135,138,148,162,172,199,208,230],"willingness":[13],"of":[14,123,128,191],"authors":[15,85,115],"to":[16,37,48,73,88,105,133,146,175,180,229],"share":[17,49],"relevant":[18],"source":[19,231],"code":[20,76,97,106,118,232],"and":[21,46,52,64,69,75,141],"data.":[22],"Ten":[23],"years":[24],"after":[25],"Ted":[26],"Pedersen\u2019s":[27],"influential":[28],"\u201cLast":[29],"Words\u201d":[30],"contribution":[31],"Computational":[33],"Linguistics,":[34],"we":[35,131,160,194],"investigate":[36],"what":[38],"extent":[39],"researchers":[40],"linguistics":[43,185],"are":[44,195],"willing":[45],"able":[47,145],"their":[50],"data":[51,74,93,108,140],"code.":[53,142],"We":[54,143],"surveyed":[55],"all":[56],"395":[57],"full":[58],"papers":[59],"presented":[60],"at":[61],"2011":[63],"2016":[65,179],"ACL":[66],"Annual":[67],"Meetings,":[68],"identified":[70],"whether":[71],"links":[72,81,104,228],"were":[77,82,86,94,109,144],"provided.":[78],"If":[79],"working":[80,103,227],"not":[83,110,204],"provided,":[84],"requested":[87],"provide":[89],"this":[90],"information.":[91],"Although":[92],"often":[95],"available,":[96],"was":[98],"shared":[99],"less":[100],"often.":[101],"When":[102],"or":[107],"provided":[111,116,139],"paper,":[114],"about":[120,198],"one":[121],"third":[122],"cases.":[124],"For":[125,154],"a":[126,156,189,211],"selection":[127],"ten":[129],"papers,":[130],"attempted":[132],"reproduce":[134,147],"results":[136,149],"using":[137],"approximately":[150],"six":[152],"papers.":[153],"only":[155,205],"single":[157],"paper":[158],"did":[159],"obtain":[161],"exact":[163],"same":[164],"results.":[165],"Our":[166],"findings":[167],"show":[168],"that":[169],"even":[170],"though":[171],"situation":[173],"appears":[174],"have":[176],"improved":[177],"comparing":[178],"2011,":[181],"empiricism":[182],"still":[186],"largely":[187],"remains":[188],"matter":[190],"faith.":[192],"Nevertheless,":[193],"somewhat":[196],"optimistic":[197],"future.":[200],"Ensuring":[201],"is":[203,233],"important":[206],"field":[209],"as":[210],"whole,":[212],"but":[213],"also":[214],"seems":[215],"worthwhile":[216],"individual":[218],"researchers:":[219],"The":[220],"median":[221],"citation":[222],"count":[223],"studies":[225],"with":[226],"higher.":[234]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":26},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":11},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":8}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-10-10T00:00:00"}