{"id":"https://openalex.org/W2756217188","doi":"https://doi.org/10.1162/coli_a_00304","title":"Sharing Is Caring: The Future of Shared Tasks","display_name":"Sharing Is Caring: The Future of Shared Tasks","publication_year":2017,"publication_date":"2017-09-11","ids":{"openalex":"https://openalex.org/W2756217188","doi":"https://doi.org/10.1162/coli_a_00304","mag":"2756217188"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00304","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00304","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00304","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00304","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040564747","display_name":"Malvina Nissim","orcid":"https://orcid.org/0000-0001-5289-0971"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Malvina Nissim","raw_affiliation_strings":["University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071765023","display_name":"Lasha Abzianidze","orcid":"https://orcid.org/0000-0003-4169-2202"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Lasha Abzianidze","raw_affiliation_strings":["University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052327595","display_name":"Kilian Evang","orcid":null},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Kilian Evang","raw_affiliation_strings":["University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039811894","display_name":"Rob van der Goot","orcid":"https://orcid.org/0009-0003-1999-4156"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Rob van der Goot","raw_affiliation_strings":["University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034237344","display_name":"Hessel Haagsma","orcid":"https://orcid.org/0000-0003-1514-072X"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Hessel Haagsma","raw_affiliation_strings":["University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088832285","display_name":"Barbara Plank","orcid":"https://orcid.org/0000-0002-4394-1965"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Barbara Plank","raw_affiliation_strings":["University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075698724","display_name":"Martijn Wieling","orcid":"https://orcid.org/0000-0003-0434-1526"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Martijn Wieling","raw_affiliation_strings":["University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Groningen, CLCG, Faculty of Arts, Oude Kijk in 't Jatstraat 26, 9712 EK Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5040564747"],"corresponding_institution_ids":["https://openalex.org/I169381384"],"apc_list":null,"apc_paid":null,"fwci":2.24,"has_fulltext":true,"cited_by_count":21,"citation_normalized_percentile":{"value":0.9078804,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"43","issue":"4","first_page":"897","last_page":"904"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.980400025844574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6526929140090942},{"id":"https://openalex.org/keywords/competitor-analysis","display_name":"Competitor analysis","score":0.649039626121521},{"id":"https://openalex.org/keywords/competition","display_name":"Competition (biology)","score":0.6201270222663879},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.5960646867752075},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5823075771331787},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5649530291557312},{"id":"https://openalex.org/keywords/harmonization","display_name":"Harmonization","score":0.4994843006134033},{"id":"https://openalex.org/keywords/position","display_name":"Position (finance)","score":0.4513664245605469},{"id":"https://openalex.org/keywords/public-relations","display_name":"Public relations","score":0.34330499172210693},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.28508150577545166},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.25692978501319885},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.20546364784240723},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.14852690696716309},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.14573520421981812},{"id":"https://openalex.org/keywords/management","display_name":"Management","score":0.13337430357933044},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.09968438744544983}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6526929140090942},{"id":"https://openalex.org/C127576917","wikidata":"https://www.wikidata.org/wiki/Q624630","display_name":"Competitor analysis","level":2,"score":0.649039626121521},{"id":"https://openalex.org/C91306197","wikidata":"https://www.wikidata.org/wiki/Q45767","display_name":"Competition (biology)","level":2,"score":0.6201270222663879},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.5960646867752075},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5823075771331787},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5649530291557312},{"id":"https://openalex.org/C2779962950","wikidata":"https://www.wikidata.org/wiki/Q5659376","display_name":"Harmonization","level":2,"score":0.4994843006134033},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.4513664245605469},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.34330499172210693},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.28508150577545166},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.25692978501319885},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.20546364784240723},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.14852690696716309},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.14573520421981812},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.13337430357933044},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.09968438744544983},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1162/coli_a_00304","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00304","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00304","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:pure.rug.nl:publications/14fcab8e-6ef1-4d80-9160-e8babe0164fe","is_oa":true,"landing_page_url":"https://research.rug.nl/en/publications/14fcab8e-6ef1-4d80-9160-e8babe0164fe","pdf_url":"https://pure.rug.nl/ws/files/51522662/COLI_a_00304.pdf","source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Nissim, M, Abzianidze, L, Evang, K, van der Goot, R, Haagsma, H, Plank, B & Wieling, M 2017, 'Sharing is Caring : The Future of Shared Tasks', Computational Linguistics, vol. 43, no. 4, pp. 897-904. https://doi.org/10.1162/COLI_a_00304","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:125d96a890f14c72918b81255a9af221","is_oa":false,"landing_page_url":"https://doaj.org/article/125d96a890f14c72918b81255a9af221","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 43, Iss 4 (2017)","raw_type":"article"},{"id":"pmh:oai:pure.rug.nl:openaire_cris_publications/14fcab8e-6ef1-4d80-9160-e8babe0164fe","is_oa":true,"landing_page_url":"https://hdl.handle.net/11370/14fcab8e-6ef1-4d80-9160-e8babe0164fe","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Nissim, M, Abzianidze, L, Evang, K, van der Goot, R, Haagsma, H, Plank, B & Wieling, M 2017, 'Sharing is Caring : The Future of Shared Tasks', Computational Linguistics, vol. 43, no. 4, pp. 897-904. https://doi.org/10.1162/COLI_a_00304","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00304","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00304","pdf_url":"https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00304","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4099999964237213,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320320933","display_name":"Rijksuniversiteit Groningen","ror":"https://ror.org/012p63287"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2756217188.pdf","grobid_xml":"https://content.openalex.org/works/W2756217188.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W86774453","https://openalex.org/W1664649366","https://openalex.org/W1975004953","https://openalex.org/W2098297786","https://openalex.org/W2250947781","https://openalex.org/W2251964473","https://openalex.org/W2397447617","https://openalex.org/W2740626591","https://openalex.org/W2917881729"],"related_works":["https://openalex.org/W2368605798","https://openalex.org/W2518037665","https://openalex.org/W2348524959","https://openalex.org/W2368049389","https://openalex.org/W2170801710","https://openalex.org/W2384861574","https://openalex.org/W2952704802","https://openalex.org/W2768297557","https://openalex.org/W2353197970","https://openalex.org/W2366991863"],"abstract_inverted_index":{"Shared":[0,1431],"tasks":[1,29,73,120,162,392,470,519,870,1033,1075,1432,1579,1625,1674,1814,2014,2246,2447,2609,3013],"are":[2,158,557,562,575,592,827,839,1027,1147,1161,1281,1316,1433,1754,1855,1887,2183,2199,2388,2434,2708,2914,2927,2935,3035,3319,3391],"indisputably":[3],"drivers":[4],"of":[5,116,258,279,296,305,314,334,369,383,416,449,492,495,504,516,545,599,630,640,675,722,768,776,782,811,868,879,886,925,929,954,970,981,990,1018,1020,1039,1056,1087,1112,1119,1132,1225,1304,1311,1330,1374,1419,1503,1512,1572,1590,1633,1643,1668,1686,1689,1703,1736,1798,1808,1846,1881,1956,2007,2029,2036,2046,2061,2068,2079,2118,2169,2222,2236,2369,2375,2378,2395,2409,2413,2475,2544,2553,2570,2626,2645,2683,2720,2744,2818,2829,2847,2924,2945,2954,2988,3020,3027,3049,3109,3119,3133,3144,3170,3231,3236,3297,3301,3309,3387],"progress":[6,1604,1985,2977,3194],"and":[7,34,40,45,59,91,127,137,149,189,224,289,310,337,367,387,406,478,482,501,583,602,874,1006,1068,1097,1143,1344,1386,1417,1439,1459,1467,1490,1613,1628,1790,1829,1865,1874,1905,1935,1959,1970,2040,2081,2143,2151,2193,2202,2265,2387,2407,2455,2460,2470,2632,2639,2670,2686,2713,2724,2734,2741,2746,2797,2895,2920,2959,2978,3006,3043,3051,3142,3159,3163,3177,3200,3234,3238,3255,3324,3330,3337,3344,3378],"interest":[8,1418,2989],"for":[9,32,47,185,331,542,559,832,876,936,951,1064,1069,1080,1089,1149,1229,1470,1559,1576,1623,1714,1982,2307,2320,2351,2437,2453,2533,2616,2717,2788,2835,2855,3011,3150,3192,3269,3315,3334,3382,3397],"problems":[10,346,547,1525,1661],"in":[11,96,108,170,196,249,290,393,421,456,520,531,550,564,597,644,673,689,715,740,770,789,992,1003,1015,1031,1071,1099,1157,1251,1298,1302,1313,1378,1389,1476,1479,1605,1626,1638,2012,2015,2017,2044,2059,2066,2167,2174,2239,2247,2287,2344,2480,2528,2549,2607,2710,2748,2763,2842,2890,2993,3038,3115,3125,3140,3195,3229,3325],"NLP.":[12,394],"This":[13,77,707,1290,1442,1474,1722,2071,2487,2525,2629,2653,2736],"is":[14,65,78,144,178,180,199,221,234,437,511,712,779,1034,1051,1060,1137,1141,1194,1206,1291,1327,1356,1364,1370,1395,1506,1532,1580,1675,1694,1710,1775,1788,1826,1834,1837,1896,1932,1979,2001,2072,2132,2141,2326,2399,2444,2654,2906,2957,2964,2990,3008,3053,3102,3106,3135,3264],"reflected":[15],"by":[16,23,92,348,506,616,805,808,909,1672,1745,1766,2095,2218,2484,2718,3094],"their":[17,441,645,941,1001,1029,1249,1257,1288,1336,1423,1452,1999,2621,2732,3208,3227,3270,3335],"increasing":[18],"popularity,":[19],"as":[20,22,133,402,428,490,655,945,1084,1301,1862,1937,2022,2197,2231,2270,2272,2335,2565,2596,2601,2603,2706,2852,2854,3032,3104,3363],"well":[21,2509],"the":[24,94,104,113,134,171,176,187,207,243,315,332,345,364,413,422,502,546,551,560,661,664,751,755,766,780,793,802,819,829,887,900,926,955,1065,1085,1100,1129,1133,1175,1222,1226,1230,1252,1277,1328,1354,1371,1414,1429,1472,1496,1504,1573,1587,1631,1660,1669,1687,1767,1782,1785,1791,1796,1799,1806,1819,1844,1879,1901,1953,1974,1990,1995,2005,2018,2033,2056,2076,2156,2175,2179,2220,2234,2240,2252,2288,2296,2308,2321,2327,2332,2341,2376,2392,2405,2424,2430,2450,2473,2481,2500,2537,2583,2624,2643,2656,2661,2672,2681,2742,2749,2756,2764,2770,2789,2816,2825,2876,2892,2908,2922,2973,3025,3028,3040,3044,3047,3057,3080,3098,3110,3131,3151,3168,3171,3212,3298,3302,3307,3312,3348,3357],"fact":[25],"that":[26,69,88,166,175,204,284,447,509,556,622,638,672,686,709,754,823,843,915,1024,1155,1171,1179,1270,1293,1320,1332,1398,1547,1663,1695,1764,1776,1835,2004,2134,2426,2464,2496,2642,2807,2863,2912,2937,2966,2999,3014,3100,3174,3211,3222],"new":[27,1130,2116,2237],"shared":[28,50,72,89,117,161,261,280,335,391,417,469,518,647,676,705,869,930,960,1032,1074,1122,1246,1514,1522,1578,1624,1673,1768,1813,2013,2030,2087,2158,2176,2245,2289,2294,2379,2396,2446,2465,2608,2821,3012,3184],"regularly":[30],"emerge":[31],"under-researched":[33],"under-resourced":[35],"topics,":[36],"especially":[37,2655,3009],"at":[38,787,822,2155,2761,2769,2875,3295,3306],"workshops":[39],"smaller":[41,704],"conferences.The":[42],"general":[43,268,529,2016],"procedures":[44],"conventions":[46],"organizing":[48,141],"a":[49,80,182,256,301,328,381,403,595,609,877,923,952,959,963,967,1011,1035,1047,1061,1120,1164,1195,1245,1361,1408,1455,1464,1484,1488,1507,1513,1535,1566,1653,1678,1683,1816,1926,1964,2042,2119,2160,2215,2255,2315,2362,2372,2516,2541,2571,2599,2613,2676,2689,2819,2830,2869,3017,3242,3277],"task":[51,118,648,961,1123,1232,1247,1515,1769,2088,2159,2177,2290,2466,2822,3202],"have":[52,657,855,1815,2024,2338,2598,2776,2809,3207,3280],"arisen":[53],"organically":[54],"over":[55,213,1127,1519,2651,3079],"time":[56,2878,2889],"(Paroubek,":[57],"Chaudiron,":[58],"Hirschman,":[60],"2007,":[61],"Section":[62,2843,3116],"7).":[63],"There":[64,838,1315],"no":[66,1309,1960],"consistent":[67],"framework":[68],"describes":[70,2620],"how":[71,322,339,439,1651],"should":[74,209,606,1425,1543,1610,1988,2090,2105,2291,2383,2468,2576,2630,2634,2648,2885,3001,3240],"be":[75,210,454,459,608,872,895,934,1214,1217,1308,1347,1400,1407,1426,1550,1611,1910,2092,2106,2292,2367,2385,2489,2530,2635,2649,2696,2704,2715,2784,2851,2868,2942,2948,3002,3091,3241],"organized.":[76],"not":[79,164,181,239,355,512,593,607,828,893,1092,1125,1138,1162,1282,1296,1334,1365,1396,1406,1517,1581,1592,1706,1755,1919,2058,2203,2301,2357,2384,2416,2478,2523,2547,2563,2618,2850,2941,2991,3086,3122,3136,3353],"harmful":[81],"thing":[82],"per":[83],"se,":[84],"but":[85,522,1595,1712,2065,2165,2513,2902],"we":[86,293,299,318,398,464,548,612,623,670,683,904,916,1153,1167,1172,1180,1294,1321,1542,1607,2023,2249,2903,3113,3219,3252,3273,3293],"believe":[87,203,446,624,917,1411,1608,2324,2463,2904,2933,2998,3221],"tasks,":[90,262,336,677,931,3185],"extension":[93],"field":[95,208,748,1681,1787,2333],"general,":[97],"would":[98,1212,1307,1723,1962,2210,2555,2664],"benefit":[99],"from":[100,471,538,1169,1200,1239,1244,1276,1891,2126,2390,2840,3249],"some":[101,295,448,544,580,852,2579,2691,2880,2928],"reflection":[102],"on":[103,247,265,307,321,486,731,765,962,1174,1359,1450,1454,1630,1701,1719,1851,1922,1943,1967,2110,2429,2506,2637,2660,2680,2800,2899,2976,3167,3183,3257,3350,3384],"existing":[105,1204],"conventions.":[106],"This,":[107],"turn,":[109,1477],"could":[110,844,947,1295,1345,1443,1636,2488,2527],"lead":[111,845,1045,1444],"to":[112,147,160,226,323,343,380,412,453,458,527,571,660,800,818,846,857,939,986,1010,1046,1128,1182,1186,1202,1216,1219,1274,1285,1341,1349,1382,1436,1445,1495,1520,1553,1570,1640,1658,1676,1728,1757,1804,1899,1947,1951,1984,2054,2075,2108,2113,2123,2137,2186,2212,2228,2232,2244,2285,2305,2330,2340,2401,2519,2557,2578,2597,2623,2730,2778,2786,2795,2812,2815,2859,2971,3004,3024,3056,3074,3097,3154,3164,3186,3206,3225,3245,3266,3275,3283,3321,3327,3347,3394],"future":[114,1260,1577,2438],"harmonization":[115],"procedures.Shared":[119],"revolve":[121],"around":[122,1586],"two":[123,156,1110,1317],"aspects:":[124],"research":[125,131,241,420,600,864,1048,1497,1603,1666,1927,1983,2647],"advancement":[126,132],"competition.":[128,2652,2974],"We":[129,201,282,360,445,484,534,1617,1648,2209,2323,2462,2674,2932,2996,3084,3367,3390],"see":[130,699],"driving":[135,3193],"force":[136,1948],"main":[138,414,1832],"goal":[139,415,2394,2644],"behind":[140,1773],"them.":[142,297,1944,2278],"Competition":[143,2841],"an":[145,513,524,833,847,949,987,1004,1077,1480,1620,1923,2493,2566,3064],"instrument":[146,2871],"encourage":[148,2665],"promote":[150],"participation.":[151],"However,":[152,298,682,1073,1351,2032,2381,3218,3272],"just":[153],"because":[154,218,435,796,1198,1353,1707,2569],"these":[155,450,2934,3033],"forces":[157],"intrinsic":[159,225,2709],"does":[163,238,1091,2300,2415,2562,3352],"mean":[165,948],"they":[167,285,791,797,1280,2182,2198,2268,2504,2707],"always":[168,1213,1889],"act":[169],"same":[172,956,1223,1227,1231,1783,1954,2771,2877,3082],"direction:":[173],"Ensuring":[174],"competition":[177,316,1253,1278,1792,2254,2662,2772],"fair":[179,215],"necessary":[183],"requirement":[184],"advancing":[186,206,899,2646,2873],"field,":[188,836,901,1067,2039],"might":[190,854,892,1044,1272,1405,1549,1698,2226,2808,2849,3015],"even":[191,1614,2490],"slow":[192],"down":[193],"progress.Our":[194],"position":[195],"this":[197,291,396,400,462,510,539,573,627,687,777,920,946,1170,1193,1404,1540,1740,1774,1823,2214,2242,2299,2325,2905,3007,3317,3388],"respect":[198,411,1494],"clear:":[200],"do":[202,1124,1333,1516,1637,1918,2448,2997,3085,3161,3220],"(i)":[205],"given":[211,1456,2820],"priority":[212],"ensuring":[214],"competition,":[216,1818],"also":[217,684,725,907,977,1596,1649,2184,2514,2753,2984,3062,3392],"(ii)":[219],"inequality":[220,652,3105],"partly":[222],"unsolvable":[223],"life.":[227],"In":[228,395,667,994,1209,1749,1802,2279],"other":[229,820,841,995,1762,1852,1875,2280,2856],"words:":[230,2281],"Equality":[231,3089],"between":[232,480,1487,1903],"competitors":[233],"desirable":[235,935,1215,2133],"if":[236,1248,1279,1777,2768,2952],"it":[237,340,436,932,1136,1211,1716,1836,1895,2104,2131,2311,2963,3070,3072,3101,3148,3364],"hinder":[240,1665],"advancement.In":[242],"recently":[244],"established":[245,1013],"workshop":[246],"ethics":[248],"NLP,1Parra":[250],"Escart\u00edn":[251,350,374,618,633,760,911,973,1103,1265,3381],"et":[252,351,375,619,634,761,784,912,974,1104,1236,1266,2100,2206],"al.":[253,352,376,620,635,762,785,913,975,1105,1267],"(2017)":[254,377,621,636,763,914,976,1106,1268],"raise":[255,286,1107],"set":[257,1189,1458,1955],"considerations":[259],"involving":[260],"mainly":[263,749],"focusing":[264,306,1448,2659],"areas":[266],"where":[267,1509,1898,2258,2503,2845],"ethical":[269,308,388,982,2701],"concerns":[270,371,430,451],"regarding":[271,1645],"good":[272],"scientific":[273,325],"practice":[274,1014],"intersect":[275],"with":[276,362,410,579,691,697,734,898,1493,1534,1602,1726,1733,1840,1858,2048,2295,2551,3260,3356],"certain":[277],"aspects":[278,1589],"tasks.":[281,706,718,1523,2031,2380,2397],"find":[283],"valid":[287,2870],"concerns,":[288,587],"contribution,":[292],"address":[294,2755],"take":[300,399,1544,2213],"different":[302,756,1690,1800,2773,2779,2801,3077],"perspective.":[303],"Instead":[304],"issues":[309,389,425,614,906,1108,2694,2702,2929,2936],"potential":[311,384,429,586,1500],"negative":[312,385,1025,1042,1058,1082,1501,1599,2442,2865,2955],"effects":[313,386],"aspect,":[317],"rather":[319,431,523,1461,2418,2980],"concentrate":[320],"bolster":[324],"progress.We":[326],"make":[327,2358,3071,3147,3187],"simple":[329,1654,2677],"proposal":[330,1575],"improvement":[333],"discuss":[338,905],"can":[341,724,871,933,1254,1322,1399,1656,1664,1848,1977,2260,2274,2366],"help":[342,1657,2556],"mitigate":[344],"raised":[347,615,908,1662],"Parra":[349,373,617,632,759,910,972,1102,1264,3380],"(2017),":[353],"while":[354],"necessarily":[356,594,894],"tackling":[357,1539],"them":[358,604,2897,3188],"directly.":[359],"start":[361],"assessing":[363,543,2891],"concrete":[365],"impact":[366,443],"significance":[368],"such":[370,489,1652,1861,1885,1908],"first.Recently,":[372],"drew":[378],"attention":[379],"list":[382,401],"concerning":[390,1109],"section,":[397],"starting":[404,752,1199],"point":[405,753,2580],"examine":[407],"each":[408,3201],"problem":[409,596,1197,1409,1693,1741,1833,2035],"tasks\u2014to":[418],"advance":[419],"field.":[423],"Some":[424,2699],"were":[426,798],"regarded":[427],"than":[432,663,1462,1763,2419,2981],"definite":[433],"problems,":[434],"unclear":[438,1897],"large":[440,1734,2373],"actual":[442,3313],"is.":[444],"needed":[452,2136],"quantified":[455],"order":[457,1639,1803],"properly":[460,2916],"assessed.To":[461],"end,":[463],"reviewed":[465],"about":[466,651,1041,1402,1555,2404,2420],"100":[467],"recent":[468,532],"various":[472,3258],"campaigns":[473],"(SemEval,":[474],"EVALITA,":[475],"CoNLL,":[476],"WMT,":[477],"CLEF)":[479],"2014":[481],"2016.":[483],"focused":[485],"several":[487],"aspects,":[488,1318],"participation":[491,494,639,711,1312,1424,2349,2837],"companies,":[493],"organizers,":[496],"closed":[497,1750,1904,2008,3060,3126,3360],"versus":[498,3359],"open":[499,3358],"tracks,":[500,1751],"submission":[503,781],"papers":[505,1088],"participants.":[507,666,1971],"Note":[508],"exhaustive":[514,2589],"overview":[515,2538,3041],"all":[517,1778,1949,2084,2172,2359,2586,2781],"NLP,":[521,1627],"arbitrary":[525],"sample":[526,1380],"investigate":[528],"trends":[530],"times.":[533],"use":[535,810,1758,1780,1952,1978,1989,2006,2178,2187,2261,3132],"information":[536,574,1358,1644,2526],"drawn":[537],"annotation":[540],"exercise":[541],"report":[549,764],"following":[552],"sections.":[553],"The":[554,567,719,1054,1324,1367,1692,1771,1831,2827,2883,2986,3291],"figures":[555],"relevant":[558,2992],"discussion":[561,3182,3292],"reported":[563,3037],"Table":[565,700,741,1390],"1.":[566],"annotated":[568],"spreadsheets":[569],"used":[570,1158,2284,2343],"collect":[572],"publicly":[576,2142,2200],"available,":[577,1993,2145,2361],"together":[578,3254],"basic":[581],"statistics":[582],"additional":[584,1809,2188],"explanations.2Some":[585],"although":[588,1393],"being":[589,2391,2793],"ethically":[590],"relevant,":[591],"terms":[598,2060,2067,2168,3230],"advancement,":[601],"fixing":[603],"directly":[605,1220,2610,2738],"priority.":[610],"Here,":[611],"assess":[613,3155],"fall":[625,918],"into":[626,919,1545,3287],"category.Potential":[628],"Conflicts":[629],"Interest.":[631],"state":[637,885,3169],"organizers":[641,656,678,728,1635,1969,2467,2534,2575,3000,3213],"or":[642,863,958,1178,1190,1483,1564,1912,1994,2355,2568,2939],"annotators":[643],"own":[646,3209],"raises":[649],"questions":[650],"among":[653],"participants,":[654,1376],"earlier":[658,3385],"access":[659,856,1727,2304,2339,2777,2796],"data":[662,1188,1437,1457,1761,1909,2532,2802],"regular":[665],"our":[668,1184,1299,1379,2027,2994,3175],"survey,":[669,1300],"found":[671],"5.8%":[674],"did":[679,2477,2522,2546],"indeed":[680,1346],"participate.":[681],"observe":[685],"happens":[688],"connection":[690,2550],"few":[692,2020,3278],"participants":[693,723,997,1447,1548,1976,2259,2497,2587,2729,2858,3224],"(average":[694],"3.5":[695],"compared":[696,733],"12.1,":[698],"1),":[701],"thus":[702,2041],"typically":[703],"indicates":[708],"organizers'":[710],"more":[713,858,1449,1527,1720,1729,2317,3190],"common":[714,2961,3243],"small,":[716],"specialized":[717],"low":[720,2457],"number":[721,1329,1373,3019],"explain":[726,2520],"why":[727,1360],"perform":[729],"better":[730,2558,3288],"average":[732,737],"non-organizers":[735],"(see":[736,2838],"normalized":[738],"rank":[739],"1).Unequal":[742],"Playing":[743,2758],"Field.":[744,1101,2750],"An":[745,773],"unequal":[746,834,848,2037],"playing":[747,835,849,1680,1786,2038,3066],"reflects":[750],"teams":[757,821,853,938,1331,1725,1753,1916,1950,1987,2047,2173,2225,2352,2774,2946,2968,3022],"have.":[758],"issue":[767,778,1055,1632,2078,2743],"differences":[769,1156],"processing":[771],"power.":[772],"extreme":[774],"example":[775,1560],"Durrani":[783],"(2013)":[786],"WMT13,":[788],"which":[790,814,1050,1339,1377,2421,3156,3160],"reached":[792],"highest":[794],"scores":[795],"able":[799,1218,2794],"boost":[801],"BLEU":[803],"score":[804],"approximately":[806,1342],"0.8%":[807],"making":[809,2251],"1TB":[812],"RAM,":[813],"was":[815,1717,2163,3311],"probably":[816,2886],"unavailable":[817],"time.":[824],"Computational":[825,3303],"resources":[826,1594,1870,1975,1992,1996,2190,2238,2283,2306,2342,2360,2782,2848,3134],"only":[830,1593,1838,2166],"reason":[831],"though.":[837,902],"many":[840,1812,3261],"causes":[842],"field\u2014for":[850],"example,":[851,1263,1715],"proprietary":[859,861,1737,2723,2831],"data,":[860,1784,1811,1842,1894,2064,2148,2264],"software,":[862],"equipment.The":[865],"competitive":[866,927],"nature":[867,928,2712,3026],"fun,":[873],"stimulating":[875],"variety":[878],"reasons":[880,891],"(visibility,":[881],"grant":[882],"applications,":[883],"beating":[884],"art,":[888],"etc.).":[889],"Such":[890],"positively":[896],"correlated":[897],"Here":[903],"category.Secretiveness.":[921],"As":[922,966,2688,3030,3112],"result":[924,1478],"participating":[937,1646,1752,2485,2606,3021],"keep":[940,2348],"\u201csecret":[942],"sauce\u201d":[943],"private,":[944],"advantage":[950],"re-run":[953],"task,":[957,1134,1177],"related":[964,1348,1521],"problem.":[965,1473,1498],"possible":[968,2350,2965,3149],"effect":[969,1807],"secretiveness,":[971],"mention":[978,3276],"\u201cUnconscious":[979],"overlooking":[980],"concerns,\u201d":[983],"actually":[984,1163],"referring":[985],"unacceptable":[988],"level":[989,1679],"vagueness":[991],"papers.":[993],"words,":[996],"may":[998,1242],"unintentionally":[999],"describe":[1000],"systems":[1002,1185,1205,1453,1779,1847,1940,2370,2733,2799,2814,2913,3145],"abstract":[1005],"vague":[1007],"way":[1008],"due":[1009,2811,3023],"previously":[1012],"systems'":[1016],"descriptions.Lack":[1017],"Description":[1019],"Negative":[1021],"Results.":[1022],"Given":[1023],"results":[1026,1043,1059,1224,1793,2089,2140,2433,2443,2866,2956,3050],"informative,":[1028],"under-representation":[1030],"concern.":[1036],"A":[1037,1499,1697],"lack":[1038],"knowledge":[1040],"redundancy,":[1049,1113],"clearly":[1052],"undesirable.":[1053],"under-represented":[1057],"global":[1062],"concern":[1063],"entire":[1066],"science":[1070],"general.":[1072],"provide":[1076,2449,2588],"excellent":[1078],"opportunity":[1079],"publishing":[1081,2441],"results,":[1083],"acceptance":[1086,2633],"publication":[1090],"particularly":[1093],"favor":[1094,1724,2045,2233],"positive":[1095],"results.Redundancy":[1096],"Replicability":[1098,2747],"types":[1111],"(a)":[1114],"when":[1115,1145,1529,2535,2658],"optimal":[1116],"parameter":[1117,1159],"settings":[1118,1160,3210],"previous":[1121,1176],"carry":[1126,1518],"version":[1131],"therefore":[1135,1618,2002],"clear":[1139,2542],"what":[1140,1634,2414,2476,2507,2521,2545,3351],"learned;":[1142],"(b)":[1144],"algorithms":[1146],"reimplemented":[1148],"replicability":[1150],"purposes.Regarding":[1151],"(a),":[1152],"think":[1154,3087],"bad":[1165],"thing;":[1166],"learn":[1168],"overfit":[1173],"need":[1181,2940],"adapt":[1183],"another":[1187],"domain.Regarding":[1191],"(b),":[1192],"real":[1196],"scratch":[1201],"reimplement":[1203,2124],"unnecessarily":[1207],"time-consuming.":[1208],"addition,":[1210],"reproduce":[1221,2138],"model":[1228],"(Pedersen,":[1233,2097],"2008;":[1234,2098],"Fokkens":[1235,2099],"al.,":[1237,2101,2207],"2013).Withdrawal":[1238],"Competition.":[1240],"Participants":[1241],"withdraw":[1243,1275],"ranking":[1250,1482,3052],"negatively":[1255],"affect":[1256],"reputation":[1258],"and/or":[1259],"funding.":[1261],"For":[1262],"suggest":[1269,1619,2675],"companies":[1271,1732],"prefer":[1273],"highly":[1283],"ranked,":[1284],"avoid":[1286],"blemishing":[1287],"reputation.":[1289],"something":[1292,2561],"quantify":[1297],"case":[1303,2657],"withdrawal":[1305],"there":[1306,1394,2926],"evidence":[1310],"reports.":[1314],"though,":[1319],"quantify.":[1323],"first":[1325],"aspect":[1326,1369,2663],"publish":[1335],"system's":[1337],"description,":[1338],"amounts":[1340,1381],"9%,":[1343],"withdrawals.":[1350],"exactly":[1352,1781],"paper":[1355,2614,3042],"missing,":[1357],"team":[1362,2422],"withdrew":[1363],"available.":[1366],"second":[1368],"total":[1372],"industry":[1375],"20%":[1383],"(\u201cSome":[1384],"company\u201d":[1385,1388],"\u201cOnly":[1387],"1).":[1391],"Thus,":[1392],"much":[1397],"done":[1401],"withdrawal\u2014and":[1403],"anyway\u2014we":[1410],"that,":[1412],"considering":[1413],"substantial":[1415],"presence":[1416,2828],"industries":[1420],"so":[1421,2152,2195,2230,2595],"far,":[1422],"accommodated.Potentially":[1427],"Gaming":[1428,2824],"System.":[1430],"usually":[1434],"bound":[1435],"sets":[1438,2803],"evaluation":[1440,2501],"metrics.":[1441],"competition-oriented":[1446],"tuning":[1451,1561],"metrics":[1460],"finding":[1463],"scientifically":[1465],"sound":[1466],"scalable":[1468],"method":[1469,1709],"solving":[1471,2879],"can,":[1475],"\u201cunfair\u201d":[1481],"misleading":[1485],"relation":[1486],"methodology":[1489],"its":[1491,1708],"value":[1492],"outcome":[1502],"latter":[1505],"scenario":[1508],"\u201coptimal\u201d":[1510],"methods":[1511,2459,3078,3120,3157],"These":[1524],"become":[1526],"severe":[1528],"system":[1530,1567,1696,1704,2425,2622,3045],"gaming":[1531,2671],"combined":[1533],"secretive":[1536],"attitude.":[1537],"While":[1538],"issue,":[1541],"account":[1546],"less":[1551],"eager":[1552],"write":[1554,2860],"ad":[1556],"hoc":[1557],"solutions,":[1558],"pre-processing":[1562],"components":[1563,1857],"tailoring":[1565,2813],"too":[1568],"closely":[1569,2073],"specifics":[1571,2817],"annotation.Our":[1574],"revolutionary.":[1582],"It":[1583,2751,3263],"simply":[1584],"revolves":[1585],"key":[1588],"sharing,":[1591,3096],"experiences,":[1597],"including":[1598,2146,2411,2592],"ones.":[1600],"Specifically,":[1601],"mind,":[1606],"sharing":[1609,1642,2221,2979],"encouraged":[1612,2958],"partially":[1615,2754],"enforced.":[1616],"explicit":[1621],"setting":[1622,3362],"reflect":[1629],"maximize":[1641],"systems.":[1647,2346],"show":[1650,1795],"strategy":[1655],"overcome":[1659],"advancement.One":[1667],"challenges":[1670],"faced":[1671],"ensure":[1677,2129,2302,2314,2581,2918],"permitting":[1682],"transparent":[1684,3124],"comparison":[1685,3118],"merits":[1688,2406],"methods.":[1691,1801],"come":[1699],"out":[1700,2115],"top":[1702],"B":[1705],"superior,":[1711],"because,":[1713,1981],"trained":[1718],"data.":[1721],"resources,":[1730,2050,2780,3233],"like":[1731,1871,1878,2211,2668],"quantities":[1735],"in-house":[1738],"data.Traditionally,":[1739],"has":[1742,2010,3121,3342],"been":[1743,2810,3123],"mitigated":[1744],"establishing":[1746],"\u201cclosed":[1747],"tracks.\u201d":[1748],"allowed":[1756,2185,3003],"any":[1759,2262],"training":[1760,1810,1841,2063],"provided":[1765],"organizers.":[1770],"rationale":[1772],"equal,":[1789],"will":[1794,2312,2337,2356,2695,2727,2737,2752,2775,2783,2804,2867,2917,2969,2983,3069,3090,3146,3179,3203,3214],"strengths":[1797],"study":[1805],"separate":[1817,2517],"so-called":[1820],"\u201copen":[1821],"track.\u201dHowever,":[1822],"open\u2013closed":[1824],"division":[1825],"increasingly":[1827,3092],"impractical":[1828,1934],"ineffective.":[1830],"concerned":[1839],"whereas":[1843],"performance":[1845,2458],"crucially":[1849],"depend":[1850,1942],"resources.":[1853,1883,2735],"Examples":[1854],"external":[1856,1892,1930,1957,2189,2627],"pre-trained":[1859,2149,2266],"models,":[1860,1958,2150],"part-of-speech":[1863],"taggers":[1864],"dependency":[1866],"parsers,":[1867],"auxiliary":[1868],"data-derived":[1869],"word":[1872],"embeddings,":[1873],"influential":[1876],"factors":[1877],"availability":[1880,2552],"computational":[1882],"Because":[1884,2605],"models":[1886,1931,2267],"almost":[1888],"derived":[1890],"language":[1893],"draw":[1900],"line":[1902],"open.":[1906],"Should":[1907],"disallowed":[1911],"not?":[1913],"If":[1914],"not,":[1915,3162],"still":[1917,2313,2833],"really":[1920],"participate":[1921,2286],"equal":[1924,2062,2069,2303,2318,3065],"footing.From":[1925],"perspective,":[1928],"banning":[1929],"completely":[1933],"nonsensical,":[1936],"most":[1938,3216],"state-of-the-art":[1939,2345],"now":[1941],"Likewise,":[1945],"trying":[1946],"other,":[1961],"place":[1963],"heavy":[1965],"burden":[1966],"both":[1968,3039],"Moreover,":[1972,2103,2540,2792],"restricting":[1973],"questionable,":[1980],"quickly,":[1986],"best":[1991,1997,2428],"fitting":[1998],"system.It":[2000],"unsurprising":[2003],"tracks":[2009,2726,3061,3127],"declined":[2011],"last":[2019],"years,":[2021],"observed":[2025],"during":[2026],"review":[2028],"original":[2034],"bias":[2043],"ample":[2049],"remains.We":[2051],"propose,":[2052],"then,":[2053],"rethink":[2055],"problem,":[2057,2760],"opportunities.":[2070],"connected":[2074],"wider":[2077],"reproducibility":[2080],"replicability:":[2082],"Like":[2083],"published":[2085],"research,":[2086,2874],"ideally":[2091],"fully":[2093],"reproducible":[2094],"anyone":[2096],"2013).3":[2102],"easy":[2107,2853],"build":[2109],"others'":[2111],"work":[2112,2479,2564,3158,3355],"try":[2114],"variations":[2117],"method,":[2120],"without":[2121],"having":[2122],"things":[2125],"scratch.":[2127],"To":[2128,2347],"this,":[2130],"everything":[2135],"experimental":[2139],"freely":[2144,2276],"code,":[2147,2263,2554],"on.":[2153],"Interestingly,":[2154],"CoNLL-2013":[2157],"similar":[2161],"step":[2162,2216,2329],"taken,":[2164],"pre-condition:":[2170],"\u201cWhile":[2171],"NUCLE":[2180],"corpus,":[2181],"(both":[2191],"corpora":[2192],"tools)":[2194],"long":[2196,2271,2765,3031,3250],"available":[2201,2785],"proprietary\u201d":[2204],"(Ng":[2205],"2013).":[2208],"further,":[2217],"enforcing":[2219,2911],"whatever":[2223],"resource":[2224,3095],"choose":[2227,2970],"use,":[2229],"injection":[2235],"field.Applying":[2241],"principle":[2243,2682],"practice,":[2248,2962],"propose":[2250],"primary":[2253],"\u201cpublic":[2256],"track,\u201d":[2257],"want,":[2269],"others":[2273],"then":[2275],"obtain":[2277],"All":[2282],"subsequently":[2293],"community.":[2297],"Although":[2298,2440],"current":[2309],"edition,":[2310],"progressively":[2316,3166],"footing":[2319],"future.":[2322],"crucial":[2328],"move":[2331],"forward,":[2334],"everyone":[2336,2619,2787,3268,3322],"who":[2353,3279],"cannot":[2354,2703,2714,2938,2947],"secondary,":[2363],"\u201cproprietary":[2364],"track\u201d":[2365],"established.Ranking":[2368],"forms":[2371],"part":[2374,3108],"appeal":[2377],"rankings":[2382],"overemphasized":[2386],"far":[2389],"final":[2393],"Research":[2398],"supposed":[2400],"teach":[2402],"us":[2403],"characteristics":[2408],"methods,":[2410],"insights":[2412],"work,":[2417,2548],"built":[2423],"performed":[2427],"test":[2431],"data.Negative":[2432],"very":[2435],"informative":[2436],"developments.":[2439],"difficult,":[2445],"ideal":[2451],"context":[2452],"disclosing":[2454],"explaining":[2456],"choices.":[2461],"explicitly":[2469,3036],"strongly":[2471],"solicit":[2472],"inclusion":[2474],"reports":[2482,2862,2894,3141],"written":[2483],"teams.":[2486],"solicited":[2491],"via":[2492],"online":[2494],"form":[2495],"submit":[2498],"after":[2499],"phase,":[2502],"comment":[2505],"worked":[2508],"(as":[2510],"commonly":[2511],"done),":[2512],"provides":[2515],"section":[2518],"work.":[2524],"turn":[2529],"valuable":[2531,3338],"compiling":[2536],"report.":[2539],"explanation":[2543],"understand":[2559],"whether":[2560],"idea":[2567],"specific":[2572,2721],"implementation.More":[2573],"generally,":[2574],"encourage\u2014and":[2577],"through":[2582],"reviewing":[2584],"process\u2014that":[2585],"reports,":[2590],"potentially":[2591],"ablation/addition":[2593],"tests,":[2594],"picture":[2600],"comprehensive":[2602,2861],"possible.":[2604],"implies":[2611],"getting":[2612],"accepted":[2615],"publication,":[2617],"satisfaction":[2625],"reviewers.":[2628],"change,":[2631],"conditional":[2636],"clarity":[2638,2900,3139],"exhaustiveness.We":[2640],"stated":[2641],"prioritized":[2650],"undesired":[2666],"practices":[2667],"secretiveness":[2669],"system.":[2673],"solution":[2678],"based":[2679],"maximizing":[2684],"resource-":[2685],"information-sharing.":[2687],"byproduct,":[2690],"problematic":[2692],"competition-related":[2693],"overcome,":[2697],"too.":[2698],"outstanding":[2700,2881],"solved,":[2705],"human":[2711],"controlled":[2716,2949],"means":[2719],"guidelines.Introducing":[2722],"public":[2725],"stimulate":[2728],"release":[2731,3143,3226],"reduce":[2739],"Secretiveness":[2740],"Redundancy":[2745],"Unequal":[2757],"Field":[2759],"least":[2762],"run:":[2766],"Even":[2767],"next":[2790],"round.":[2791],"run":[2798],"uncover":[2805],"limitations":[2806],"(Potential":[2823],"System).":[2826],"track":[2832,3361],"allows":[2834],"industrial":[2836],"Withdrawal":[2839,2944],"2.2),":[2844],"distribution":[2846],"teams.Encouraging":[2857],"include":[2864],"towards":[2872],"problems.":[2882],"reviewers":[2884],"spend":[2887],"extra":[2888],"single":[2893],"accept":[2896],"conditionally":[2898],"requirements,":[2901],"worth":[2907],"effort.":[2909],"Indeed,":[2910],"described":[2915]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":1}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}