{"id":"https://openalex.org/W2153072998","doi":"https://doi.org/10.1162/089120104773633402","title":"The Kappa Statistic: A Second Look","display_name":"The Kappa Statistic: A Second Look","publication_year":2004,"publication_date":"2004-03-01","ids":{"openalex":"https://openalex.org/W2153072998","doi":"https://doi.org/10.1162/089120104773633402","mag":"2153072998"},"language":"en","primary_location":{"id":"doi:10.1162/089120104773633402","is_oa":true,"landing_page_url":"https://doi.org/10.1162/089120104773633402","pdf_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/089120104773633402","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/089120104773633402","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055258367","display_name":"Barbara Di Eugenio","orcid":"https://orcid.org/0000-0003-1706-2577"},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Barbara Di Eugenio","raw_affiliation_strings":["University of Illinois at Chicago, Computer Science, 1120 SEO (M/C 152), 851 South Morgan Street, Chicago, IL 60607","University of Illinois at Chicago, Computer Science, 1120 SEO (M/C 152), 851 South Morgan Street, Chicago, IL 60607. E-mail:"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Chicago, Computer Science, 1120 SEO (M/C 152), 851 South Morgan Street, Chicago, IL 60607","institution_ids":["https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Illinois at Chicago, Computer Science, 1120 SEO (M/C 152), 851 South Morgan Street, Chicago, IL 60607. E-mail:","institution_ids":["https://openalex.org/I39422238"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019777435","display_name":"Michael Glass","orcid":"https://orcid.org/0009-0000-1505-4667"},"institutions":[{"id":"https://openalex.org/I149744451","display_name":"Valparaiso University","ror":"https://ror.org/01pp0fx48","country_code":"US","type":"education","lineage":["https://openalex.org/I149744451"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Michael Glass","raw_affiliation_strings":["Valparaiso University, Mathematics and Computer Science, 116 Gellerson Hall, Valparaiso, IN 46383","Valparaiso University, Mathematics and Computer Science, 116 Gellerson Hall, Valparaiso, IN 46383. E-mail:"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Valparaiso University, Mathematics and Computer Science, 116 Gellerson Hall, Valparaiso, IN 46383","institution_ids":["https://openalex.org/I149744451"]},{"raw_affiliation_string":"Valparaiso University, Mathematics and Computer Science, 116 Gellerson Hall, Valparaiso, IN 46383. E-mail:","institution_ids":["https://openalex.org/I149744451"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5019777435","https://openalex.org/A5055258367"],"corresponding_institution_ids":["https://openalex.org/I149744451","https://openalex.org/I39422238"],"apc_list":null,"apc_paid":null,"fwci":33.0431,"has_fulltext":true,"cited_by_count":498,"citation_normalized_percentile":{"value":0.99815974,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"30","issue":"1","first_page":"95","last_page":"101"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10050","display_name":"Multi-Criteria Decision Making","score":0.9458000063896179,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.9448000192642212,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cohens-kappa","display_name":"Cohen's kappa","score":0.8546061515808105},{"id":"https://openalex.org/keywords/statistic","display_name":"Statistic","score":0.6974107027053833},{"id":"https://openalex.org/keywords/kappa","display_name":"Kappa","score":0.6830229759216309},{"id":"https://openalex.org/keywords/de-facto","display_name":"De facto","score":0.6664798259735107},{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.6643217206001282},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5657692551612854},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5469220280647278},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.5375878810882568},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4938717484474182},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.4562368094921112},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.4247174561023712},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.253822922706604},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2450326383113861},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.2309485375881195},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.15782079100608826},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.12802466750144958},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.09552744030952454},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.08328944444656372},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.058664798736572266}],"concepts":[{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.8546061515808105},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.6974107027053833},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.6830229759216309},{"id":"https://openalex.org/C2992317946","wikidata":"https://www.wikidata.org/wiki/Q712144","display_name":"De facto","level":2,"score":0.6664798259735107},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.6643217206001282},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5657692551612854},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5469220280647278},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5375878810882568},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4938717484474182},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4562368094921112},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.4247174561023712},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.253822922706604},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2450326383113861},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2309485375881195},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.15782079100608826},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.12802466750144958},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.09552744030952454},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.08328944444656372},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.058664798736572266},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/089120104773633402","is_oa":true,"landing_page_url":"https://doi.org/10.1162/089120104773633402","pdf_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/089120104773633402","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:scholar.valpo.edu:mcs_fac_pubs-1017","is_oa":false,"landing_page_url":"http://scholar.valpo.edu/mcs_fac_pubs/18","pdf_url":null,"source":{"id":"https://openalex.org/S4306401623","display_name":"ValpoScholar (Valparaiso University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I149744451","host_organization_name":"Valparaiso University","host_organization_lineage":["https://openalex.org/I149744451"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Mathematics and Computer Science Faculty Publications","raw_type":"text"},{"id":"pmh:oai:doaj.org/article:3737158097ce46c38cbfb15ab1595622","is_oa":false,"landing_page_url":"https://doaj.org/article/3737158097ce46c38cbfb15ab1595622","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 30, Iss 1 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/089120104773633402","is_oa":true,"landing_page_url":"https://doi.org/10.1162/089120104773633402","pdf_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/089120104773633402","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2153072998.pdf","grobid_xml":"https://content.openalex.org/works/W2153072998.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1536975410","https://openalex.org/W1583650837","https://openalex.org/W1970543122","https://openalex.org/W1975879668","https://openalex.org/W2002664886","https://openalex.org/W2025638903","https://openalex.org/W2053154970","https://openalex.org/W2070285512","https://openalex.org/W2076061140","https://openalex.org/W2109730238","https://openalex.org/W2133943399","https://openalex.org/W2144489542","https://openalex.org/W2150098611","https://openalex.org/W2151528607","https://openalex.org/W2153222072","https://openalex.org/W2153804780","https://openalex.org/W2408074938","https://openalex.org/W2907147533","https://openalex.org/W4231741839"],"related_works":["https://openalex.org/W1967594194","https://openalex.org/W1444168786","https://openalex.org/W2294084971","https://openalex.org/W2148964467","https://openalex.org/W2281321759","https://openalex.org/W2474048269","https://openalex.org/W2792864698","https://openalex.org/W1966976587","https://openalex.org/W2292870489","https://openalex.org/W4294214983"],"abstract_inverted_index":{"In":[0,21],"recent":[1],"years,":[2],"the":[3,10,32,40,46,60],"kappa":[4],"coefficient":[5],"of":[6,45,50],"agreement":[7,17,48],"has":[8,34],"become":[9],"de":[11],"facto":[12],"standard":[13],"for":[14,18],"evaluating":[15],"intercoder":[16],"tagging":[19],"tasks.":[20],"this":[22],"squib,":[23],"we":[24,38,53],"highlight":[25],"issues":[26],"that":[27,31],"affect":[28,59],"\u03ba":[29,61],"and":[30,57],"community":[33],"largely":[35],"neglected.":[36],"First,":[37],"discuss":[39,54],"assumptions":[41],"underlying":[42],"different":[43],"computations":[44],"expected":[47],"component":[49],"\u03ba.":[51],"Second,":[52],"how":[55],"prevalence":[56],"bias":[58],"measure.":[62]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":23},{"year":2022,"cited_by_count":24},{"year":2021,"cited_by_count":29},{"year":2020,"cited_by_count":24},{"year":2019,"cited_by_count":15},{"year":2018,"cited_by_count":15},{"year":2017,"cited_by_count":26},{"year":2016,"cited_by_count":28},{"year":2015,"cited_by_count":31},{"year":2014,"cited_by_count":30},{"year":2013,"cited_by_count":25},{"year":2012,"cited_by_count":30}],"updated_date":"2026-08-12T07:12:00.856984","created_date":"2025-10-10T00:00:00"}