{"id":"https://openalex.org/W4303614602","doi":"https://doi.org/10.1162/coli_a_00464","title":"Annotation Error Detection: Analyzing the Past and Present for a More Coherent Future","display_name":"Annotation Error Detection: Analyzing the Past and Present for a More Coherent Future","publication_year":2022,"publication_date":"2022-10-07","ids":{"openalex":"https://openalex.org/W4303614602","doi":"https://doi.org/10.1162/coli_a_00464"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00464","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00464","pdf_url":"https://direct.mit.edu/coli/article-pdf/49/1/157/2068980/coli_a_00464.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/coli/article-pdf/49/1/157/2068980/coli_a_00464.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070380110","display_name":"Jan-Christoph Klie","orcid":"https://orcid.org/0000-0003-0181-6450"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Jan-Christoph Klie","raw_affiliation_strings":["Ubiquitous Knowledge Processing Lab, Department of Computer Science, Technical University of Darmstadt. www.ukp.tu-darmstadt.de"],"affiliations":[{"raw_affiliation_string":"Ubiquitous Knowledge Processing Lab, Department of Computer Science, Technical University of Darmstadt. www.ukp.tu-darmstadt.de","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089032031","display_name":"Bonnie Webber","orcid":"https://orcid.org/0000-0002-4284-8216"},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Bonnie Webber","raw_affiliation_strings":["School of Informatics,, University of Edinburgh"],"affiliations":[{"raw_affiliation_string":"School of Informatics,, University of Edinburgh","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027450194","display_name":"Iryna Gurevych","orcid":"https://orcid.org/0000-0003-2187-7621"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Iryna Gurevych","raw_affiliation_strings":["UKP Lab / TU Darmstadt"],"affiliations":[{"raw_affiliation_string":"UKP Lab / TU Darmstadt","institution_ids":["https://openalex.org/I31512782"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5070380110"],"corresponding_institution_ids":["https://openalex.org/I31512782"],"apc_list":null,"apc_paid":null,"fwci":4.0259,"has_fulltext":true,"cited_by_count":32,"citation_normalized_percentile":{"value":0.94499012,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"49","issue":"1","first_page":"157","last_page":"198"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8698312640190125},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8556678295135498},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.5558128952980042},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5482296943664551},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5196520090103149},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.49113723635673523},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.48094743490219116},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.47626087069511414},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45367997884750366},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4245196282863617},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4232971668243408},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.41498127579689026},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40735000371932983},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.39637458324432373},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07609939575195312}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8698312640190125},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8556678295135498},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5558128952980042},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5482296943664551},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5196520090103149},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.49113723635673523},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48094743490219116},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.47626087069511414},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45367997884750366},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4245196282863617},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4232971668243408},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.41498127579689026},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40735000371932983},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.39637458324432373},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07609939575195312},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/coli_a_00464","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00464","pdf_url":"https://direct.mit.edu/coli/article-pdf/49/1/157/2068980/coli_a_00464.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:c2208525153f468a9148bd1a812b3032","is_oa":true,"landing_page_url":"https://doaj.org/article/c2208525153f468a9148bd1a812b3032","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 49, Iss 1 (2023)","raw_type":"article"},{"id":"pmh:oai:tubiblio.ulb.tu-darmstadt.de:134422","is_oa":false,"landing_page_url":"http://tubiblio.ulb.tu-darmstadt.de/134422/","pdf_url":null,"source":{"id":"https://openalex.org/S4377196390","display_name":"TUbilio (Technical University of Darmstadt)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I31512782","host_organization_name":"Technische Universit\u00e4t Darmstadt","host_organization_lineage":["https://openalex.org/I31512782"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Artikel"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00464","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00464","pdf_url":"https://direct.mit.edu/coli/article-pdf/49/1/157/2068980/coli_a_00464.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.7900000214576721,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4303614602.pdf","grobid_xml":"https://content.openalex.org/works/W4303614602.grobid-xml"},"referenced_works_count":117,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W131522978","https://openalex.org/W569478347","https://openalex.org/W1515847863","https://openalex.org/W1516184288","https://openalex.org/W1528012351","https://openalex.org/W1583837637","https://openalex.org/W1594247117","https://openalex.org/W1618905105","https://openalex.org/W1962580118","https://openalex.org/W1966716734","https://openalex.org/W1985514943","https://openalex.org/W2006873020","https://openalex.org/W2012942264","https://openalex.org/W2021664962","https://openalex.org/W2051834357","https://openalex.org/W2076018040","https://openalex.org/W2077302143","https://openalex.org/W2082024724","https://openalex.org/W2084398551","https://openalex.org/W2086613190","https://openalex.org/W2105408267","https://openalex.org/W2127937179","https://openalex.org/W2144182447","https://openalex.org/W2144578941","https://openalex.org/W2155923110","https://openalex.org/W2165507745","https://openalex.org/W2250493512","https://openalex.org/W2250539671","https://openalex.org/W2250600206","https://openalex.org/W2250872259","https://openalex.org/W2252039660","https://openalex.org/W2299976354","https://openalex.org/W2476974423","https://openalex.org/W2499811908","https://openalex.org/W2531638282","https://openalex.org/W2602331152","https://openalex.org/W2608702473","https://openalex.org/W2614908767","https://openalex.org/W2799915114","https://openalex.org/W2804447254","https://openalex.org/W2891177506","https://openalex.org/W2899689163","https://openalex.org/W2912083425","https://openalex.org/W2912145841","https://openalex.org/W2914584698","https://openalex.org/W2915480215","https://openalex.org/W2928125049","https://openalex.org/W2945094081","https://openalex.org/W2962843773","https://openalex.org/W2963626623","https://openalex.org/W2965373594","https://openalex.org/W2970352191","https://openalex.org/W2970641574","https://openalex.org/W2971039193","https://openalex.org/W2972573244","https://openalex.org/W2985347336","https://openalex.org/W2989812388","https://openalex.org/W3021332602","https://openalex.org/W3034238904","https://openalex.org/W3034891697","https://openalex.org/W3042609801","https://openalex.org/W3087927280","https://openalex.org/W3099982175","https://openalex.org/W3103649165","https://openalex.org/W3104382126","https://openalex.org/W3110687497","https://openalex.org/W3113657720","https://openalex.org/W3125278790","https://openalex.org/W3153723500","https://openalex.org/W3156669901","https://openalex.org/W3166716308","https://openalex.org/W3169654142","https://openalex.org/W3175866539","https://openalex.org/W3176174048","https://openalex.org/W3176214425","https://openalex.org/W3177468934","https://openalex.org/W3184904351","https://openalex.org/W4213009331","https://openalex.org/W4231040763","https://openalex.org/W4233787372","https://openalex.org/W4239372721","https://openalex.org/W4243479147","https://openalex.org/W4250561792","https://openalex.org/W4252684946","https://openalex.org/W4391156274","https://openalex.org/W6607126429","https://openalex.org/W6617145748","https://openalex.org/W6633503733","https://openalex.org/W6635717444","https://openalex.org/W6636501900","https://openalex.org/W6644307160","https://openalex.org/W6679805048","https://openalex.org/W6680323016","https://openalex.org/W6681190473","https://openalex.org/W6685053522","https://openalex.org/W6691296735","https://openalex.org/W6691459498","https://openalex.org/W6691601049","https://openalex.org/W6691692454","https://openalex.org/W6691768687","https://openalex.org/W6728622933","https://openalex.org/W6732382073","https://openalex.org/W6739651123","https://openalex.org/W6743611570","https://openalex.org/W6745159025","https://openalex.org/W6745609711","https://openalex.org/W6752143646","https://openalex.org/W6755207826","https://openalex.org/W6766673545","https://openalex.org/W6768851824","https://openalex.org/W6770813465","https://openalex.org/W6773894483","https://openalex.org/W6775935039","https://openalex.org/W6782932637","https://openalex.org/W6793012311","https://openalex.org/W6800807386"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W4295769391","https://openalex.org/W3123033100","https://openalex.org/W2009704030","https://openalex.org/W2245373004"],"abstract_inverted_index":{"Abstract":[0],"Annotated":[1],"data":[2],"is":[3,19],"an":[4,173],"essential":[5],"ingredient":[6],"in":[7,172],"natural":[8],"language":[9],"processing":[10],"for":[11,23,55,114,126],"training":[12],"and":[13,99,107,119,133,156,164,170,175],"evaluating":[14],"machine":[15],"learning":[16],"models.":[17],"It":[18],"therefore":[20,110],"very":[21],"desirable":[22],"the":[24,63,88,149],"annotations":[25],"to":[26,83,103],"be":[27],"of":[28,44,148],"high":[29],"quality.":[30],"Recent":[31],"work,":[32],"however,":[33],"has":[34],"shown":[35],"that":[36,68],"several":[37],"popular":[38],"datasets":[39,125,169],"contain":[40],"a":[41,140,145],"surprising":[42],"number":[43],"annotation":[45,56,117,150],"errors":[46,118],"or":[47,86],"inconsistencies.":[48],"To":[49,160],"alleviate":[50],"this":[51],"issue,":[52],"many":[53],"methods":[54,82,113],"error":[57,151],"detection":[58,152],"have":[59],"been":[60],"devised":[61],"over":[62],"years.":[64],"While":[65],"researchers":[66],"show":[67],"their":[69,74,81,105],"approaches":[70],"work":[71,85],"well":[72,130],"on":[73,87,95,122],"newly":[75],"introduced":[76],"datasets,":[77],"they":[78],"rarely":[79],"compare":[80],"previous":[84],"same":[89],"datasets.":[90],"This":[91],"raises":[92],"strong":[93],"concerns":[94],"methods\u2019":[96],"general":[97,157],"performance":[98],"makes":[100],"it":[101],"difficult":[102],"assess":[104],"strengths":[106],"weaknesses.":[108],"We":[109],"reimplement":[111],"18":[112],"detecting":[115],"potential":[116],"evaluate":[120],"them":[121],"9":[123],"English":[124],"text":[127],"classification":[128],"as":[129,131],"token":[132],"span":[134],"labeling.":[135],"In":[136],"addition,":[137],"we":[138,166],"define":[139],"uniform":[141],"evaluation":[142,154],"setup":[143],"including":[144],"new":[146],"formalization":[147],"task,":[153],"protocol,":[155],"best":[158],"practices.":[159],"facilitate":[161],"future":[162],"research":[163],"reproducibility,":[165],"release":[167],"our":[168],"implementations":[171],"easy-to-use":[174],"open":[176],"source":[177],"software":[178],"package.1":[179]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":2}],"updated_date":"2026-04-06T07:47:59.780226","created_date":"2025-10-10T00:00:00"}