{"id":"https://openalex.org/W2899689163","doi":"https://doi.org/10.1162/tacl_a_00040","title":"Comparing Bayesian Models of Annotation","display_name":"Comparing Bayesian Models of Annotation","publication_year":2018,"publication_date":"2018-12-01","ids":{"openalex":"https://openalex.org/W2899689163","doi":"https://doi.org/10.1162/tacl_a_00040","mag":"2899689163"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00040","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00040","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00040/1567662/tacl_a_00040.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00040/1567662/tacl_a_00040.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019552564","display_name":"Silviu Paun","orcid":null},"institutions":[{"id":"https://openalex.org/I166337079","display_name":"Queen Mary University of London","ror":"https://ror.org/026zzn846","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I166337079"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Silviu Paun","raw_affiliation_strings":["School of Electronic Engineering and Computer Science, Queen Mary University of London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electronic Engineering and Computer Science, Queen Mary University of London","institution_ids":["https://openalex.org/I166337079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102808840","display_name":"Bob Carpenter","orcid":"https://orcid.org/0000-0002-2433-9688"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bob Carpenter","raw_affiliation_strings":["Department of Statistics, Columbia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics, Columbia University","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067137759","display_name":"Jon Chamberlain","orcid":"https://orcid.org/0000-0002-6947-8964"},"institutions":[{"id":"https://openalex.org/I110002522","display_name":"University of Essex","ror":"https://ror.org/02nkf1q06","country_code":"GB","type":"education","lineage":["https://openalex.org/I110002522"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jon Chamberlain","raw_affiliation_strings":["School of Computer Science and Electronic Engineering, University of Essex"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Electronic Engineering, University of Essex","institution_ids":["https://openalex.org/I110002522"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084505122","display_name":"Dirk Hovy","orcid":"https://orcid.org/0000-0002-4618-3127"},"institutions":[{"id":"https://openalex.org/I71209653","display_name":"Bocconi University","ror":"https://ror.org/05crjpb27","country_code":"IT","type":"education","lineage":["https://openalex.org/I71209653"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Dirk Hovy","raw_affiliation_strings":["Department of Marketing, Bocconi University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Marketing, Bocconi University","institution_ids":["https://openalex.org/I71209653"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014534985","display_name":"Udo Kruschwitz","orcid":"https://orcid.org/0000-0002-5503-0341"},"institutions":[{"id":"https://openalex.org/I110002522","display_name":"University of Essex","ror":"https://ror.org/02nkf1q06","country_code":"GB","type":"education","lineage":["https://openalex.org/I110002522"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Udo Kruschwitz","raw_affiliation_strings":["School of Computer Science and Electronic Engineering, University of Essex"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Electronic Engineering, University of Essex","institution_ids":["https://openalex.org/I110002522"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047065550","display_name":"Massimo Poesio","orcid":"https://orcid.org/0000-0001-8469-2072"},"institutions":[{"id":"https://openalex.org/I166337079","display_name":"Queen Mary University of London","ror":"https://ror.org/026zzn846","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I166337079"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Massimo Poesio","raw_affiliation_strings":["School of Electronic Engineering and Computer Science, Queen Mary University of London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electronic Engineering and Computer Science, Queen Mary University of London","institution_ids":["https://openalex.org/I166337079"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.3291,"has_fulltext":false,"cited_by_count":82,"citation_normalized_percentile":{"value":0.96505959,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"6","issue":null,"first_page":"571","last_page":"585"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9886000156402588,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9876999855041504,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8757612705230713},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.742544949054718},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7420397996902466},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.5727012753486633},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5707054138183594},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.554222822189331},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.4658224284648895},{"id":"https://openalex.org/keywords/tying","display_name":"Tying","score":0.45969951152801514},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.4480833411216736},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32724320888519287}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8757612705230713},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.742544949054718},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7420397996902466},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.5727012753486633},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5707054138183594},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.554222822189331},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.4658224284648895},{"id":"https://openalex.org/C2780938662","wikidata":"https://www.wikidata.org/wiki/Q973710","display_name":"Tying","level":2,"score":0.45969951152801514},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4480833411216736},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32724320888519287},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1162/tacl_a_00040","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00040","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00040/1567662/tacl_a_00040.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:epub.uni-regensburg.de:40304","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401229","display_name":"University of Regensburg Publication Server (University of Regensburg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I60668342","host_organization_name":"University of Regensburg","host_organization_lineage":["https://openalex.org/I60668342"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:94a446b92fdc4652a75eb0ad3b12a40e","is_oa":false,"landing_page_url":"https://doaj.org/article/94a446b92fdc4652a75eb0ad3b12a40e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 6 (2021)","raw_type":"article"},{"id":"pmh:oai:qmro.qmul.ac.uk:123456789/55140","is_oa":false,"landing_page_url":"https://qmro.qmul.ac.uk/xmlui/handle/123456789/55140","pdf_url":null,"source":{"id":"https://openalex.org/S4306400530","display_name":"Queen Mary Research Online (Queen Mary University of London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I166337079","host_organization_name":"Queen Mary University of London","host_organization_lineage":["https://openalex.org/I166337079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00040","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00040","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00040/1567662/tacl_a_00040.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W172170795","https://openalex.org/W1459599406","https://openalex.org/W1486839939","https://openalex.org/W1501025848","https://openalex.org/W1510659740","https://openalex.org/W1543648998","https://openalex.org/W1839751919","https://openalex.org/W1857053476","https://openalex.org/W1877880381","https://openalex.org/W1970381522","https://openalex.org/W1981457167","https://openalex.org/W1983897914","https://openalex.org/W1988520084","https://openalex.org/W1990190154","https://openalex.org/W2035866663","https://openalex.org/W2058753770","https://openalex.org/W2067750885","https://openalex.org/W2082802837","https://openalex.org/W2087101057","https://openalex.org/W2121856130","https://openalex.org/W2127374659","https://openalex.org/W2130715829","https://openalex.org/W2134305421","https://openalex.org/W2140787629","https://openalex.org/W2141766660","https://openalex.org/W2142518823","https://openalex.org/W2144660879","https://openalex.org/W2148534890","https://openalex.org/W2157010107","https://openalex.org/W2203714058","https://openalex.org/W2250202591","https://openalex.org/W2250854390","https://openalex.org/W2251054028","https://openalex.org/W2251818274","https://openalex.org/W2252039660","https://openalex.org/W2295951612","https://openalex.org/W2296107862","https://openalex.org/W2562522356","https://openalex.org/W2572405493","https://openalex.org/W2575233155","https://openalex.org/W2577537660","https://openalex.org/W2592805751","https://openalex.org/W2963677523","https://openalex.org/W3020204246"],"related_works":["https://openalex.org/W2998037107","https://openalex.org/W1569871744","https://openalex.org/W2379437105","https://openalex.org/W2016024526","https://openalex.org/W3139928442","https://openalex.org/W1567336638","https://openalex.org/W3124892642","https://openalex.org/W3125367266","https://openalex.org/W1654902617","https://openalex.org/W2095430756"],"abstract_inverted_index":{"The":[0],"analysis":[1,46],"of":[2,38,47,81,126,131],"crowdsourced":[3],"annotations":[4,49],"in":[5,128],"natural":[6],"language":[7],"processing":[8],"is":[9],"concerned":[10],"with":[11,123,137],"identifying":[12],"(1)":[13],"gold":[14,108],"standard":[15],"labels,":[16,109],"(2)":[17],"annotator":[18,87,115],"accuracies":[19],"and":[20,22,26,36,42,91,97,117,143],"biases,":[21],"(3)":[23],"item":[24,89,118],"difficulties":[25],"error":[27],"patterns.":[28],"Traditionally,":[29],"majority":[30],"voting":[31],"was":[32],"used":[33],"for":[34,40,112,139],"1,":[35],"coefficients":[37],"agreement":[39],"2":[41],"3.":[43],"Lately,":[44],"model-based":[45],"corpus":[48],"have":[50],"proven":[51],"better":[52],"at":[53],"all":[54],"three":[55],"tasks.":[56],"But":[57],"there":[58],"has":[59],"been":[60],"relatively":[61],"little":[62],"work":[63],"comparing":[64],"them":[65],"on":[66],"the":[67,129],"same":[68],"datasets.":[69],"This":[70],"paper":[71],"aims":[72],"to":[73,86,107],"fill":[74],"this":[75],"gap":[76],"by":[77],"analyzing":[78],"six":[79],"models":[80,102],"annotation,":[82],"covering":[83],"different":[84],"approaches":[85],"ability,":[88],"difficulty,":[90,119],"parameter":[92],"pooling":[93],"(tying)":[94],"across":[95],"annotators":[96],"items.":[98],"We":[99,135],"evaluate":[100],"these":[101],"along":[103],"four":[104,121],"aspects:":[105],"comparison":[106],"predictive":[110],"accuracy":[111],"new":[113],"annotations,":[114],"characterization,":[116],"using":[120],"datasets":[122],"varying":[124],"degrees":[125],"noise":[127],"form":[130],"random":[132],"(spammy)":[133],"annotators.":[134],"conclude":[136],"guidelines":[138],"model":[140],"selection,":[141],"application,":[142],"implementation.":[144]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":18},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":1}],"updated_date":"2026-07-16T13:24:37.021932","created_date":"2025-10-10T00:00:00"}