{"id":"https://openalex.org/W2250745056","doi":"https://doi.org/10.3115/v1/e14-1058","title":"Weighted Krippendorff's alpha is a more reliable metrics for multi-coders ordinal annotations: experimental studies on emotion, opinion and coreference annotation","display_name":"Weighted Krippendorff's alpha is a more reliable metrics for multi-coders ordinal annotations: experimental studies on emotion, opinion and coreference annotation","publication_year":2014,"publication_date":"2014-01-01","ids":{"openalex":"https://openalex.org/W2250745056","doi":"https://doi.org/10.3115/v1/e14-1058","mag":"2250745056"},"language":"en","primary_location":{"id":"doi:10.3115/v1/e14-1058","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/e14-1058","pdf_url":"https://aclanthology.org/E14-1058.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th Conference of the European Chapter of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/E14-1058.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072193848","display_name":"Jean-Yves Antoine","orcid":"https://orcid.org/0000-0002-6028-1663"},"institutions":[{"id":"https://openalex.org/I110017253","display_name":"Universit\u00e9 de Tours","ror":"https://ror.org/02wwzvj46","country_code":"FR","type":"education","lineage":["https://openalex.org/I110017253"]},{"id":"https://openalex.org/I4210136805","display_name":"Laboratoire d'Informatique Fondamentale et Appliqu\u00e9e de Tours","ror":"https://ror.org/03r938j26","country_code":"FR","type":"facility","lineage":["https://openalex.org/I110017253","https://openalex.org/I4210136805"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Jean-Yves Antoine","raw_affiliation_strings":["Universit Franois Rabelais de Tours, LI (EA 6300) Blois, France","Laboratoire d'Informatique Fondamentale et Appliqu\u00e9e de Tours"],"affiliations":[{"raw_affiliation_string":"Universit Franois Rabelais de Tours, LI (EA 6300) Blois, France","institution_ids":["https://openalex.org/I110017253"]},{"raw_affiliation_string":"Laboratoire d'Informatique Fondamentale et Appliqu\u00e9e de Tours","institution_ids":["https://openalex.org/I4210136805"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000999625","display_name":"Jeanne Villaneau","orcid":null},"institutions":[{"id":"https://openalex.org/I2802204017","display_name":"Universit\u00e9 de Bretagne Sud","ror":"https://ror.org/04ed7fw48","country_code":"FR","type":"education","lineage":["https://openalex.org/I2802204017"]},{"id":"https://openalex.org/I4210099266","display_name":"Centre Hospitalier de Bretagne Sud","ror":"https://ror.org/011jgpb07","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210099266"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Jeanne Villaneau","raw_affiliation_strings":["Universit Europenne de Bretagne, IRISA Lorient, France","SEarch, Analyze, Synthesize and Interact with Data Ecosystems"],"affiliations":[{"raw_affiliation_string":"Universit Europenne de Bretagne, IRISA Lorient, France","institution_ids":["https://openalex.org/I2802204017","https://openalex.org/I4210099266"]},{"raw_affiliation_string":"SEarch, Analyze, Synthesize and Interact with Data Ecosystems","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085335345","display_name":"Ana\u00efs Lefeuvre","orcid":null},"institutions":[{"id":"https://openalex.org/I110017253","display_name":"Universit\u00e9 de Tours","ror":"https://ror.org/02wwzvj46","country_code":"FR","type":"education","lineage":["https://openalex.org/I110017253"]},{"id":"https://openalex.org/I4210142254","display_name":"Laboratoire Bordelais de Recherche en Informatique","ror":"https://ror.org/03adqg323","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I15057530","https://openalex.org/I4210142254","https://openalex.org/I4210159245","https://openalex.org/I4210160189"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Ana\u00efs Lefeuvre","raw_affiliation_strings":["Universit Franois Rabelais de Tours, LI (EA 6300) Blois, France","Laboratoire Bordelais de Recherche en Informatique"],"affiliations":[{"raw_affiliation_string":"Universit Franois Rabelais de Tours, LI (EA 6300) Blois, France","institution_ids":["https://openalex.org/I110017253"]},{"raw_affiliation_string":"Laboratoire Bordelais de Recherche en Informatique","institution_ids":["https://openalex.org/I4210142254"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5072193848"],"corresponding_institution_ids":["https://openalex.org/I110017253","https://openalex.org/I4210136805"],"apc_list":null,"apc_paid":null,"fwci":4.34682947,"has_fulltext":true,"cited_by_count":65,"citation_normalized_percentile":{"value":0.94198488,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11063","display_name":"Rough Sets and Fuzzy Logic","score":0.968999981880188,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/coreference","display_name":"Coreference","score":0.8149152398109436},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7493950128555298},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7219921350479126},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7066828608512878},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6310766935348511},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5871538519859314},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5360463261604309},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.4769391119480133},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4579739272594452},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.4173593819141388},{"id":"https://openalex.org/keywords/resolution","display_name":"Resolution (logic)","score":0.39302369952201843},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.2180558741092682},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.18325448036193848},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.15902841091156006}],"concepts":[{"id":"https://openalex.org/C28076734","wikidata":"https://www.wikidata.org/wiki/Q63087","display_name":"Coreference","level":3,"score":0.8149152398109436},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7493950128555298},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7219921350479126},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7066828608512878},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6310766935348511},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5871538519859314},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5360463261604309},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.4769391119480133},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4579739272594452},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.4173593819141388},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.39302369952201843},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2180558741092682},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.18325448036193848},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.15902841091156006},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.3115/v1/e14-1058","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/e14-1058","pdf_url":"https://aclanthology.org/E14-1058.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th Conference of the European Chapter of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.3115/v1/e14-1058","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/e14-1058","pdf_url":"https://aclanthology.org/E14-1058.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 14th Conference of the European Chapter of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6899999976158142}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2250745056.pdf","grobid_xml":"https://content.openalex.org/works/W2250745056.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W1501669607","https://openalex.org/W1568955415","https://openalex.org/W1964469912","https://openalex.org/W1972586296","https://openalex.org/W1975879668","https://openalex.org/W1983897914","https://openalex.org/W1992119553","https://openalex.org/W2003123824","https://openalex.org/W2005311637","https://openalex.org/W2007450092","https://openalex.org/W2012378416","https://openalex.org/W2018570802","https://openalex.org/W2022204871","https://openalex.org/W2037789405","https://openalex.org/W2053154970","https://openalex.org/W2058787788","https://openalex.org/W2061504941","https://openalex.org/W2070285512","https://openalex.org/W2102998034","https://openalex.org/W2115696196","https://openalex.org/W2133943399","https://openalex.org/W2141766660","https://openalex.org/W2149628368","https://openalex.org/W2153072998","https://openalex.org/W2153222072","https://openalex.org/W2153804780","https://openalex.org/W2158428778","https://openalex.org/W2159306398","https://openalex.org/W2159509436","https://openalex.org/W2165570175","https://openalex.org/W2168493061","https://openalex.org/W2170021941","https://openalex.org/W2250702579","https://openalex.org/W2333916262","https://openalex.org/W2731484148","https://openalex.org/W2949998441","https://openalex.org/W3146306708","https://openalex.org/W4239946314","https://openalex.org/W4243989635"],"related_works":["https://openalex.org/W3215967424","https://openalex.org/W2396571892","https://openalex.org/W3120396479","https://openalex.org/W2251351510","https://openalex.org/W4361282947","https://openalex.org/W1570025324","https://openalex.org/W2116902754","https://openalex.org/W3041549465","https://openalex.org/W2149586401","https://openalex.org/W2380610138"],"abstract_inverted_index":{"The":[0],"question":[1],"of":[2,6,13,66,73,76,87],"data":[3],"reliability":[4,24,81],"is":[5,21,101],"first":[7],"importance":[8],"to":[9],"assess":[10],"the":[11,22,64,85,102],"quality":[12],"manually":[14],"annotated":[15],"corpora.Although":[16],"Cohen":[17],"'":[18,50],"s":[19,51],"\u03ba":[20],"prevailing":[23],"measure":[25,90],"used":[26],"in":[27],"NLP,":[28],"alternative":[29],"statistics":[30],"have":[31],"been":[32],"proposed.This":[33],"paper":[34],"presents":[35],"an":[36,108],"experimental":[37],"study":[38],"with":[39],"four":[40],"measures":[41],"(Cohen's":[42],"\u03ba,":[43],"Scott's":[44],"\u03c0,":[45],"binary":[46],"and":[47,58],"weighted":[48,89,99],"Krippendorff":[49],"\u03b1)":[52],"on":[53,94],"three":[54],"tasks:":[55],"emotion,":[56],"opinion":[57],"coreference":[59],"annotation.The":[60],"reported":[61],"studies":[62],"investigate":[63],"factors":[65],"influence":[67,93],"(annotator":[68],"bias,":[69],"category":[70],"prevalence,":[71],"number":[72,75],"coders,":[74],"categories)":[77],"that":[78,84,98],"should":[79],"affect":[80],"estimation.Results":[82],"show":[83],"use":[86],"a":[88],"restricts":[91],"this":[92],"ordinal":[95],"annotations.They":[96],"suggest":[97],"\u03b1":[100],"most":[103],"reliable":[104],"metrics":[105],"for":[106],"such":[107],"annotation":[109],"scheme.":[110]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":13},{"year":2018,"cited_by_count":9},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":5},{"year":2015,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}