{"id":"https://openalex.org/W2019201090","doi":"https://doi.org/10.1080/00273171.2013.830064","title":"The Effect of the Raters' Marginal Distributions on Their Matched Agreement: A Rescaling Framework for Interpreting Kappa","display_name":"The Effect of the Raters' Marginal Distributions on Their Matched Agreement: A Rescaling Framework for Interpreting Kappa","publication_year":2013,"publication_date":"2013-11-01","ids":{"openalex":"https://openalex.org/W2019201090","doi":"https://doi.org/10.1080/00273171.2013.830064","mag":"2019201090","pmid":"https://pubmed.ncbi.nlm.nih.gov/26745599"},"language":"en","primary_location":{"id":"doi:10.1080/00273171.2013.830064","doi":"https://doi.org/10.1080/00273171.2013.830064","is_oa":false,"landing_page_url":"https://doi.org/10.1080/00273171.2013.830064","pdf_url":null,"source":{"id":"https://openalex.org/S64250036","display_name":"Multivariate Behavioral Research","issn_l":"0027-3171","issn":["0027-3171","1532-7906"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","dongbi-b","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Multivariate Behavioral Research","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030843228","display_name":"Tzur M. Karelitz","orcid":"https://orcid.org/0000-0003-1831-8079","observed_orcids":["https://orcid.org/0000-0003-1831-8079"]},"institutions":[{"id":"https://openalex.org/I4210134650","display_name":"National Institute for Testing and Evaluation","ror":"https://ror.org/03pm0a810","country_code":"IL","type":"nonprofit","lineage":["https://openalex.org/I4210134650"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Tzur M. Karelitz","raw_affiliation_strings":["a National Institute for Testing and Evaluation , Jerusalem , Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"a National Institute for Testing and Evaluation , Jerusalem , Israel","institution_ids":["https://openalex.org/I4210134650"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063869991","display_name":"David V. Budescu","orcid":"https://orcid.org/0000-0001-9613-0317","observed_orcids":["https://orcid.org/0000-0001-9613-0317"]},"institutions":[{"id":"https://openalex.org/I164389053","display_name":"Fordham University","ror":"https://ror.org/03qnxaf80","country_code":"US","type":"education","lineage":["https://openalex.org/I164389053"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David V. Budescu","raw_affiliation_strings":["b Department of Psychology , Fordham University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"b Department of Psychology , Fordham University","institution_ids":["https://openalex.org/I164389053"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7352,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.76988356,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":96},"biblio":{"volume":"48","issue":"6","first_page":"923","last_page":"952"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cohen-s-kappa","display_name":"Cohen's kappa","score":0.9940000176429749},{"id":"https://openalex.org/keywords/interobserver-agreement","display_name":"interobserver agreement","score":0.8500000238418579},{"id":"https://openalex.org/keywords/marginal-distributions","display_name":"marginal distributions","score":0.8679999709129333},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.5329999923706055},{"id":"https://openalex.org/keywords/essay-scoring","display_name":"essay scoring","score":0.6349999904632568}],"concepts":[{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.8706536889076233},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.6144647002220154},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5104514360427856},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.44732141494750977},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4305698275566101},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4199843108654022},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3917604088783264},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3334294557571411},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.1297507882118225},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1080/00273171.2013.830064","doi":"https://doi.org/10.1080/00273171.2013.830064","is_oa":false,"landing_page_url":"https://doi.org/10.1080/00273171.2013.830064","pdf_url":null,"source":{"id":"https://openalex.org/S64250036","display_name":"Multivariate Behavioral Research","issn_l":"0027-3171","issn":["0027-3171","1532-7906"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","dongbi-b","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Multivariate Behavioral Research","raw_type":"journal-article"},{"id":"pmid:26745599","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/26745599","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Multivariate behavioral research","raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W197081157","https://openalex.org/W625387107","https://openalex.org/W1500217743","https://openalex.org/W1559017864","https://openalex.org/W1881569770","https://openalex.org/W1970036587","https://openalex.org/W1970826185","https://openalex.org/W1976866489","https://openalex.org/W1979773093","https://openalex.org/W1983897914","https://openalex.org/W1992541562","https://openalex.org/W1995945562","https://openalex.org/W2008977378","https://openalex.org/W2009918288","https://openalex.org/W2016785976","https://openalex.org/W2018570802","https://openalex.org/W2024201314","https://openalex.org/W2031738304","https://openalex.org/W2045917085","https://openalex.org/W2048839183","https://openalex.org/W2052748753","https://openalex.org/W2053154970","https://openalex.org/W2055008194","https://openalex.org/W2055111983","https://openalex.org/W2056829027","https://openalex.org/W2059957083","https://openalex.org/W2064848794","https://openalex.org/W2065612900","https://openalex.org/W2067140887","https://openalex.org/W2070285512","https://openalex.org/W2072741316","https://openalex.org/W2076983043","https://openalex.org/W2091584422","https://openalex.org/W2101712836","https://openalex.org/W2117579985","https://openalex.org/W2124181495","https://openalex.org/W2133943399","https://openalex.org/W2149490731","https://openalex.org/W2150290224","https://openalex.org/W2153942520","https://openalex.org/W2154062297","https://openalex.org/W2164777277","https://openalex.org/W2165570175","https://openalex.org/W3026721701","https://openalex.org/W3106889297","https://openalex.org/W4210488825","https://openalex.org/W4233639736","https://openalex.org/W4240033880","https://openalex.org/W4246056272","https://openalex.org/W4253801746"],"related_works":["https://openalex.org/W2106989261","https://openalex.org/W7098572426","https://openalex.org/W2071725775","https://openalex.org/W2166440088","https://openalex.org/W2076898331","https://openalex.org/W7045428088","https://openalex.org/W1978217317","https://openalex.org/W1540240814","https://openalex.org/W2954305321","https://openalex.org/W2921171925"],"abstract_inverted_index":{"Cohen's":[0],"\u03ba":[1,74,80],"measures":[2,113],"the":[3,13,32,35,48,54,59,71,122,128,131,150],"improvement":[4],"in":[5,81,85,100,206],"classification":[6],"above":[7,44],"chance":[8],"level":[9,37,133,151],"and":[10,45,47,76,96,103,168,188,200],"it":[11],"is":[12,22,41],"most":[14],"popular":[15],"measure":[16,88,106,149],"of":[17,38,53,58,73,114,125,134,152,160,171,179,197,204],"interjudge":[18],"agreement.":[19],"Yet,":[20],"there":[21],"considerable":[23],"confusion":[24],"about":[25],"its":[26],"interpretation.":[27],"Specifically,":[28],"researchers":[29],"often":[30],"ignore":[31],"fact":[33],"that":[34,63,89,143,173],"observed":[36],"matched":[39,153],"agreement":[40,112,135,154,180],"bounded":[42],"from":[43,191],"below":[46],"bounds":[49,65],"are":[50],"a":[51,87,108,157,166,169],"function":[52],"particular":[55,158],"marginal":[56,118,139,161],"distributions":[57],"table.":[60],"We":[61,141,182],"propose":[62],"these":[64],"should":[66,145],"be":[67,146],"used":[68,147],"to":[69,148],"rescale":[70],"components":[72],"(observed":[75],"expected":[77],"agreement).":[78],"Rescaling":[79],"this":[82],"manner":[83],"results":[84],"\u03ba',":[86],"was":[90,97],"originally":[91],"proposed":[92],"by":[93],"Cohen":[94],"(1960)":[95],"largely":[98],"ignored":[99],"both":[101],"research":[102],"practice.":[104],"This":[105],"provides":[107,165],"common":[109],"scale":[110],"for":[111],"tables":[115],"with":[116,186],"different":[117],"distributions.":[119,162],"It":[120],"reaches":[121],"maximal":[123],"value":[124],"1":[126],"when":[127],"judges":[129],"show":[130],"highest":[132],"possible,":[136],"given":[137],"their":[138],"disagreements.":[140],"conclude":[142],"\u03ba'":[144],"contingent":[155],"on":[156],"set":[159,170],"The":[163],"article":[164],"framework":[167],"guidelines":[172],"facilitate":[174],"comparisons":[175],"between":[176],"various":[177],"types":[178],"tables.":[181],"illustrate":[183],"our":[184],"points":[185],"simulations":[187],"real":[189],"data":[190],"two":[192],"studies-one":[193],"involving":[194,202],"judges'":[195],"ratings":[196,203],"baseball":[198],"players":[199],"one":[201],"essays":[205],"high-stakes":[207],"tests.":[208]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2014,"cited_by_count":2}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2016-06-24T00:00:00"}