{"id":"https://openalex.org/W4407565208","doi":"https://doi.org/10.3390/make7010018","title":"Weighted Kappa for Interobserver Agreement and Missing Data","display_name":"Weighted Kappa for Interobserver Agreement and Missing Data","publication_year":2025,"publication_date":"2025-02-14","ids":{"openalex":"https://openalex.org/W4407565208","doi":"https://doi.org/10.3390/make7010018"},"language":"en","primary_location":{"id":"doi:10.3390/make7010018","is_oa":true,"landing_page_url":"https://doi.org/10.3390/make7010018","pdf_url":"https://www.mdpi.com/2504-4990/7/1/18/pdf?version=1739538659","source":{"id":"https://openalex.org/S4210213891","display_name":"Machine Learning and Knowledge Extraction","issn_l":"2504-4990","issn":["2504-4990"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning and Knowledge Extraction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2504-4990/7/1/18/pdf?version=1739538659","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026965468","display_name":"Matthijs J. Warrens","orcid":"https://orcid.org/0000-0002-7302-640X"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Matthijs J. Warrens","raw_affiliation_strings":["GION Education/Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Rozenstraat 3, 9712 TG Groningen, The Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-7302-640X","affiliations":[{"raw_affiliation_string":"GION Education/Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Rozenstraat 3, 9712 TG Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082724104","display_name":"Alexandra de Raadt","orcid":null},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Alexandra de Raadt","raw_affiliation_strings":["GION Education/Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Rozenstraat 3, 9712 TG Groningen, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"GION Education/Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Rozenstraat 3, 9712 TG Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048263863","display_name":"Roel Bosker","orcid":"https://orcid.org/0000-0002-1495-7298"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Roel J. Bosker","raw_affiliation_strings":["GION Education/Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Rozenstraat 3, 9712 TG Groningen, The Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-1495-7298","affiliations":[{"raw_affiliation_string":"GION Education/Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Rozenstraat 3, 9712 TG Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065114159","display_name":"Henk A. L. Kiers","orcid":"https://orcid.org/0000-0002-4995-9349"},"institutions":[{"id":"https://openalex.org/I169381384","display_name":"University of Groningen","ror":"https://ror.org/012p63287","country_code":"NL","type":"education","lineage":["https://openalex.org/I169381384"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Henk A. L. Kiers","raw_affiliation_strings":["Heymans Institute for Psychological Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Kruisstraat 2/1, 9712 TS Groningen, The Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-4995-9349","affiliations":[{"raw_affiliation_string":"Heymans Institute for Psychological Research, Faculty of Behavioural and Social Sciences, University of Groningen, Grote Kruisstraat 2/1, 9712 TS Groningen, The Netherlands","institution_ids":["https://openalex.org/I169381384"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5026965468"],"corresponding_institution_ids":["https://openalex.org/I169381384"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":1.864,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.86320955,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"7","issue":"1","first_page":"18","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12443","display_name":"Delphi Technique in Research","score":0.9577000141143799,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kappa","display_name":"Kappa","score":0.8947241902351379},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.688021719455719},{"id":"https://openalex.org/keywords/cohens-kappa","display_name":"Cohen's kappa","score":0.6812512874603271},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.6574576497077942},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.6066541075706482},{"id":"https://openalex.org/keywords/ordinal-data","display_name":"Ordinal data","score":0.5630079507827759},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.5387727618217468},{"id":"https://openalex.org/keywords/mean-squared-error","display_name":"Mean squared error","score":0.4849567413330078},{"id":"https://openalex.org/keywords/ordinal-scale","display_name":"Ordinal Scale","score":0.4728719890117645},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.415226012468338},{"id":"https://openalex.org/keywords/predictive-value","display_name":"Predictive value","score":0.4109220504760742},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.35844868421554565},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3141307234764099},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.15325883030891418}],"concepts":[{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.8947241902351379},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.688021719455719},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.6812512874603271},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.6574576497077942},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.6066541075706482},{"id":"https://openalex.org/C85461838","wikidata":"https://www.wikidata.org/wiki/Q7100785","display_name":"Ordinal data","level":2,"score":0.5630079507827759},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5387727618217468},{"id":"https://openalex.org/C139945424","wikidata":"https://www.wikidata.org/wiki/Q1940696","display_name":"Mean squared error","level":2,"score":0.4849567413330078},{"id":"https://openalex.org/C2909711754","wikidata":"https://www.wikidata.org/wiki/Q7100785","display_name":"Ordinal Scale","level":2,"score":0.4728719890117645},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.415226012468338},{"id":"https://openalex.org/C3019719930","wikidata":"https://www.wikidata.org/wiki/Q3910099","display_name":"Predictive value","level":2,"score":0.4109220504760742},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35844868421554565},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3141307234764099},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.15325883030891418},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.3390/make7010018","is_oa":true,"landing_page_url":"https://doi.org/10.3390/make7010018","pdf_url":"https://www.mdpi.com/2504-4990/7/1/18/pdf?version=1739538659","source":{"id":"https://openalex.org/S4210213891","display_name":"Machine Learning and Knowledge Extraction","issn_l":"2504-4990","issn":["2504-4990"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning and Knowledge Extraction","raw_type":"journal-article"},{"id":"pmh:oai:pure.rug.nl:publications/f2fdbe5a-8cc9-4913-9ffe-4bfb8eb23ebe","is_oa":true,"landing_page_url":"https://research.rug.nl/en/publications/f2fdbe5a-8cc9-4913-9ffe-4bfb8eb23ebe","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Warrens, M J, de Raadt, A, Bosker, R & Kiers, H A L 2025, 'Weighted Kappa for Interobserver Agreement and Missing Data', Machine Learning & Knowledge Extraction, vol. 7, no. 1, 18. https://doi.org/10.3390/make7010018","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:doaj.org/article:0bb90c6b45c14d119825d48fcede0a4e","is_oa":false,"landing_page_url":"https://doaj.org/article/0bb90c6b45c14d119825d48fcede0a4e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Machine Learning and Knowledge Extraction, Vol 7, Iss 1, p 18 (2025)","raw_type":"article"},{"id":"pmh:oai:pure.rug.nl:openaire/f2fdbe5a-8cc9-4913-9ffe-4bfb8eb23ebe","is_oa":true,"landing_page_url":"https://hdl.handle.net/11370/f2fdbe5a-8cc9-4913-9ffe-4bfb8eb23ebe","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Warrens, M J, de Raadt, A, Bosker, R & Kiers, H A L 2025, 'Weighted Kappa for Interobserver Agreement and Missing Data', Machine Learning & Knowledge Extraction, vol. 7, no. 1, 18. https://doi.org/10.3390/make7010018","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.3390/make7010018","is_oa":true,"landing_page_url":"https://doi.org/10.3390/make7010018","pdf_url":"https://www.mdpi.com/2504-4990/7/1/18/pdf?version=1739538659","source":{"id":"https://openalex.org/S4210213891","display_name":"Machine Learning and Knowledge Extraction","issn_l":"2504-4990","issn":["2504-4990"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning and Knowledge Extraction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407565208.pdf"},"referenced_works_count":72,"referenced_works":["https://openalex.org/W612972800","https://openalex.org/W1941999681","https://openalex.org/W1976386573","https://openalex.org/W1977098485","https://openalex.org/W1979317641","https://openalex.org/W1979773093","https://openalex.org/W1984934816","https://openalex.org/W1986691668","https://openalex.org/W1995545091","https://openalex.org/W2001388444","https://openalex.org/W2015671107","https://openalex.org/W2026310535","https://openalex.org/W2037789405","https://openalex.org/W2040872221","https://openalex.org/W2044517432","https://openalex.org/W2048839183","https://openalex.org/W2052587030","https://openalex.org/W2053154970","https://openalex.org/W2061273963","https://openalex.org/W2065199005","https://openalex.org/W2068331431","https://openalex.org/W2071152788","https://openalex.org/W2080938179","https://openalex.org/W2087642051","https://openalex.org/W2095752327","https://openalex.org/W2100358124","https://openalex.org/W2112844397","https://openalex.org/W2113559481","https://openalex.org/W2116814040","https://openalex.org/W2118415569","https://openalex.org/W2118502261","https://openalex.org/W2124127323","https://openalex.org/W2125677766","https://openalex.org/W2128639372","https://openalex.org/W2130090495","https://openalex.org/W2130836428","https://openalex.org/W2133012565","https://openalex.org/W2134843796","https://openalex.org/W2139075905","https://openalex.org/W2141218183","https://openalex.org/W2142704653","https://openalex.org/W2148534043","https://openalex.org/W2151004739","https://openalex.org/W2157707003","https://openalex.org/W2164178120","https://openalex.org/W2164777277","https://openalex.org/W2165859062","https://openalex.org/W2170674956","https://openalex.org/W2235853597","https://openalex.org/W2500279295","https://openalex.org/W2511585989","https://openalex.org/W2591767537","https://openalex.org/W2604258000","https://openalex.org/W2607507174","https://openalex.org/W2772803673","https://openalex.org/W2800968938","https://openalex.org/W2904561288","https://openalex.org/W2909761571","https://openalex.org/W2909895730","https://openalex.org/W2943647645","https://openalex.org/W2955443275","https://openalex.org/W2962594047","https://openalex.org/W3159595972","https://openalex.org/W4294214983","https://openalex.org/W4298826872","https://openalex.org/W4316038467","https://openalex.org/W4383180932","https://openalex.org/W4399641977","https://openalex.org/W6628505203","https://openalex.org/W6651427760","https://openalex.org/W6656394400","https://openalex.org/W6679809462"],"related_works":["https://openalex.org/W141250983","https://openalex.org/W1967594194","https://openalex.org/W1444168786","https://openalex.org/W2294084971","https://openalex.org/W2116323989","https://openalex.org/W2148964467","https://openalex.org/W2281321759","https://openalex.org/W2474048269","https://openalex.org/W2792864698","https://openalex.org/W2018848060"],"abstract_inverted_index":{"The":[0],"weighted":[1,33,54,80],"kappa":[2],"coefficient":[3],"is":[4,19],"commonly":[5],"used":[6],"for":[7,39],"assessing":[8],"agreement":[9,69],"between":[10],"two":[11,88],"raters":[12],"on":[13,29],"an":[14],"ordinal":[15],"scale.":[16],"This":[17],"study":[18],"the":[20,24,30,86],"first":[21],"to":[22],"assess":[23],"impact":[25],"of":[26,32,56,74,79,96,107],"missing":[27,41,65],"data":[28,42,66],"value":[31],"kappa.":[34,58,81],"We":[35,59],"compared":[36,60],"three":[37,64],"methods":[38,89],"handling":[40],"in":[43,90,94,102,105],"a":[44,53],"simulation":[45],"study:":[46],"predictive":[47],"mean":[48,83,98],"matching,":[49],"listwise":[50],"deletion":[51],"and":[52,76,101],"version":[55],"Gwet\u2019s":[57],"their":[61],"performances":[62],"under":[63],"mechanisms,":[67],"using":[68],"tables":[70],"with":[71],"various":[72],"numbers":[73],"categories":[75],"different":[77],"values":[78],"Predictive":[82],"matching":[84],"outperformed":[85],"other":[87],"most":[91],"simulated":[92],"cases":[93,104],"terms":[95,106],"root":[97],"squared":[99],"error":[100],"all":[103],"bias.":[108]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-18T10:00:31.954636","created_date":"2025-10-10T00:00:00"}