{"id":"https://openalex.org/W7134825483","doi":"https://doi.org/10.48550/arxiv.2603.06865","title":"Counting on Consensus: Selecting the Right Inter-annotator Agreement Metric for NLP Annotation and Evaluation","display_name":"Counting on Consensus: Selecting the Right Inter-annotator Agreement Metric for NLP Annotation and Evaluation","publication_year":2026,"publication_date":"2026-03-06","ids":{"openalex":"https://openalex.org/W7134825483","doi":"https://doi.org/10.48550/arxiv.2603.06865"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2603.06865","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122536672","display_name":"Joseph James","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"James, Joseph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5122536672"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.20667471,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.2849000096321106,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.2849000096321106,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.25429999828338623,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.05400000140070915,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.9158999919891357},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.7396000027656555},{"id":"https://openalex.org/keywords/categorical-variable","display_name":"Categorical variable","score":0.7225000262260437},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6897000074386597},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.652899980545044},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.517799973487854},{"id":"https://openalex.org/keywords/interval","display_name":"Interval (graph theory)","score":0.3314000070095062}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.9158999919891357},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.7396000027656555},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.7225000262260437},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6897000074386597},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6789000034332275},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6665999889373779},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.652899980545044},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5976999998092651},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.517799973487854},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3626999855041504},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.3314000070095062},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3050999939441681},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.2964000105857849},{"id":"https://openalex.org/C193429382","wikidata":"https://www.wikidata.org/wiki/Q232405","display_name":"Metric system","level":2,"score":0.2867000102996826},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.27219998836517334},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2669999897480011},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.260699987411499}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2603.06865","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2603.06865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.06865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2603.06865","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"score":0.40711840987205505,"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Human":[0],"annotation":[1,16,131],"remains":[2],"the":[3,60,100,106],"foundation":[4],"of":[5,64,102,108],"reliable":[6],"and":[7,17,30,52,56,62,74,82,96,105,121,128,132],"interpretable":[8],"data":[9,84],"in":[10,134],"Natural":[11],"Language":[12],"Processing":[13],"(NLP).":[14],"As":[15],"evaluation":[18,133],"tasks":[19],"continue":[20],"to":[21,26,114],"expand,":[22],"from":[23],"categorical":[24],"labelling":[25],"segmentation,":[27],"subjective":[28],"judgment,":[29],"continuous":[31],"rating,":[32],"measuring":[33],"agreement":[34,47,69,123],"between":[35],"annotators":[36],"has":[37,49],"become":[38],"increasingly":[39],"more":[40,126],"complex.":[41],"This":[42],"paper":[43,112],"outlines":[44],"how":[45,76],"inter-annotator":[46],"(IAA)":[48],"been":[50],"conceptualised":[51],"applied":[53],"across":[54],"NLP":[55],"related":[57],"disciplines,":[58],"describing":[59],"assumptions":[61],"limitations":[63],"common":[65],"approaches.":[66],"We":[67],"organise":[68],"measures":[70],"by":[71],"task":[72],"type":[73],"discuss":[75],"factors":[77],"such":[78],"as":[79,116],"label":[80],"imbalance":[81],"missing":[83],"influence":[85],"reliability":[86],"estimates.":[87],"In":[88],"addition,":[89],"we":[90],"highlight":[91],"best":[92],"practices":[93],"for":[94,119],"clear":[95],"transparent":[97],"reporting,":[98],"including":[99],"use":[101],"confidence":[103],"intervals":[104],"analysis":[107],"disagreement":[109],"patterns.":[110],"The":[111],"aims":[113],"serve":[115],"a":[117],"guide":[118],"selecting":[120],"interpreting":[122],"measures,":[124],"promoting":[125],"consistent":[127],"reproducible":[129],"human":[130],"NLP.":[135]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-03-11T00:00:00"}