{"id":"https://openalex.org/W30390149","doi":null,"title":"Unweighted and Weighted Kappa as Measures of Agreement for Multiple Judges","display_name":"Unweighted and Weighted Kappa as Measures of Agreement for Multiple Judges","publication_year":2009,"publication_date":"2009-08-01","ids":{"openalex":"https://openalex.org/W30390149","mag":"30390149"},"language":"en","primary_location":{"id":"mag:30390149","doi":null,"is_oa":false,"landing_page_url":"https://www.questia.com/library/journal/1P3-1874986591/unweighted-and-weighted-kappa-as-measures-of-agreement","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International journal of management","raw_type":null},"volume":null,"type":"article","indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113795238","display_name":"Paul W. Mielke","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul W. Mielke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108722414","display_name":"Kenneth J. Berry","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kenneth J. Berry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5017870887","display_name":"Janis E. Johnston","orcid":"https://orcid.org/0000-0003-1037-8237","observed_orcids":["https://orcid.org/0000-0003-1037-8237"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Janis E. Johnston","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.825,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.75653035,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"26","issue":"2","first_page":"213","last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13500","display_name":"Probability and Statistical Research","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14406","display_name":"Multidisciplinary Science and Engineering Research","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agreement","display_name":"agreement","score":0.8140000104904175},{"id":"https://openalex.org/keywords/weighted-kappa","display_name":"weighted kappa","score":1.0},{"id":"https://openalex.org/keywords/categorical-data","display_name":"categorical data","score":0.5329999923706055},{"id":"https://openalex.org/keywords/ordinal-categorical-data","display_name":"ordered categorical data","score":0.7680000066757202},{"id":"https://openalex.org/keywords/chance-corrected-agreement","display_name":"chance-corrected agreement","score":0.7689999938011169},{"id":"https://openalex.org/keywords/variance-estimation","display_name":"variance estimation","score":0.6899999976158142},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.5460000038146973},{"id":"https://openalex.org/keywords/promotion-evaluation","display_name":"promotion evaluation","score":0.43299999833106995}],"concepts":[{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.6697658896446228},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6681600213050842},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.6297921538352966},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5835062861442566},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.5675612092018127},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.48484575748443604},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.4218143820762634},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.3583357334136963},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:30390149","doi":null,"is_oa":false,"landing_page_url":"https://www.questia.com/library/journal/1P3-1874986591/unweighted-and-weighted-kappa-as-measures-of-agreement","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International journal of management","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2079713508","https://openalex.org/W6973581249","https://openalex.org/W2271016566","https://openalex.org/W2003055974","https://openalex.org/W2087651910","https://openalex.org/W2063612286","https://openalex.org/W7110697646","https://openalex.org/W1982515483","https://openalex.org/W2015793039","https://openalex.org/W1977333241"],"abstract_inverted_index":{"Unweighted":[0],"and":[1,20,44,52,60,84,91,199,236,248,258,285,297,320,340,378,399,403,462,485,549,562,578,597,607],"weighted":[2,21,45,277,321,341,534,560,570,579],"kappa":[3,22,46,216,269,278,322,342,358,368,407,494,535,571],"are":[4,47,64,323,448,483,585,610],"widely":[5,227],"used":[6,228,325],"to":[7,23,98,113,131,194,304,307,326,343,356,410,438,458,510,553,612,617],"measure":[8,114,134,150,327],"the":[9,115,124,135,151,280,328,336,444,512,555,576,602,605,608],"degree":[10,329],"of":[11,18,38,80,111,121,126,129,140,153,158,173,275,282,330,338,363,413,417,428,479,514,525,557,569,590,604],"agreement":[12,33,100,116,136,152,175,184,191,202,292,331],"between":[13,185,332],"two":[14,186,333],"independent":[15,334],"judges.":[16,39,105,574],"Extension":[17],"unweighted":[19,43,168,268,319,339,481,493,517,526,577],"three":[24,344],"or":[25,133,149,345,422],"more":[26,346],"judges":[27,51,141,187,347,497,538,616],"has":[28,348,354],"traditionally":[29],"involved":[30],"measuring":[31],"pairwise":[32,55,366,445,472,480],"among":[34,101,117,137,273],"all":[35,361],"possible":[36,127,162],"pairs":[37,362,412],"In":[40,474],"this":[41,583],"paper,":[42],"defined":[48],"for":[49,103,160,176,229,289,360,453,495,536,587],"multiple":[50,104,430,496,537,573],"compared":[53],"with":[54,311,471,572],"kappa.":[56],"Also,":[57],"exact":[58,547],"variance":[59,548],"resampling":[61,507,550],"permutation":[62,508,551],"procedures":[63,452,552],"described":[65,484],"that":[66,195,467],"yield":[67],"approximate":[68],"probability":[69,441,446,455,513,556],"values.":[70],"(ProQuest:":[71],"...":[72],"denotes":[73],"formulae":[74],"omitted.)":[75],"1:":[76],"Introduction":[77],"The":[78,300,432],"classification":[79],"objects":[81],"into":[82],"categories":[83,86,596],"ordered":[85,230,595],"is":[87,95,180,192,203,226,293,303,408,434,468],"common":[88],"in":[89,123,582],"business":[90],"management":[92],"research.":[93],"It":[94],"sometimes":[96],"important":[97],"assess":[99],"classifications":[102],"For":[106],"example,":[107],"it":[108],"may":[109],"be":[110],"interest":[112],"a":[118,138,156,161,170,477,490,506,531],"committee":[119],"comprised":[120],"upper-management":[122],"evaluation":[125],"promotions":[128],"managers":[130,154],"vice-president,":[132],"panel":[139],"rating":[142],"Small":[143],"Business":[144],"Innovation":[145],"Research":[146],"(SBIR)":[147],"proposals,":[148],"assessing":[155],"group":[157],"interns":[159],"permanent":[163],"position.":[164],"Cohen":[165],"(1960)":[166],"introduced":[167,492,581],"kappa,":[169,518,527,561],"chance-corrected":[171],"index":[172],"interjudge":[174],"categorical":[177,231],"variables.":[178],"Kappa":[179],"1":[181,220,224],"when":[182,190,201,291,436],"perfect":[183],"occurs,":[188],"0":[189],"equal":[193],"expected":[196,206],"under":[197],"independence,":[198],"negative":[200],"less":[204],"than":[205],"by":[207],"chance":[208],"(Fleiss":[209],"et":[210,218,240,252,264,387,391,395,499,540],"al.,":[211,219,241,253,265,388,392,396,500,541],"2003,":[212,254],"p.":[213,255],"434).":[214],"Weighted":[215],"(Spitzer":[217],"967":[221],";":[222,371,374,381],"Cohen,":[223],"968)":[225],"data":[232],"(Cicchetti,":[233],"1981;":[234,238],"Kramer":[235,377],"Feinstein,":[237,379],"Banerjee":[239],"1999;":[242],"Kingman,":[243],"2002;":[244,246,250],"Ludbrook,":[245],"Perkins":[247],"Becker,":[249],"Fleiss":[251],"608;":[256],"Kundel":[257,398],"Polansky,":[259,400],"2003;":[260,401],"Schuster,":[261],"2004;":[262],"Berry":[263],"2005).":[266],"Whereas":[267],"does":[270],"not":[271,294,449,469],"distinguish":[272],"degrees":[274],"disagreement,":[276],"incorporates":[279],"magnitude":[281],"each":[283,308],"disagreement":[284,309],"provides":[286,565],"partial":[287],"credit":[288],"disagreements":[290],"complete":[295],"(Maclure":[296],"Willett,":[298],"1987).":[299],"usual":[301],"approach":[302,353],"assign":[305],"weights":[306,313],"pair":[310],"larger":[312],"indicating":[314],"greater":[315],"disagreement.":[316],"While":[317],"both":[318],"conventionally":[324],"judges,":[335,364,601],"extension":[337],"been":[349,355],"problematic.":[350],"One":[351],"popular":[352],"compute":[357],"coefficients":[359,426],"i.e.,":[365],"interobserver":[367],"(Fleiss,":[369],"1971":[370,373],"Light,":[372],"Conger,":[375],"1980;":[376],"1981":[380],"Schouten,":[382],"1980,":[383],"1982a,":[384],"1982b;":[385],"Epstein":[386],"1986;":[389],"Herman":[390],"1990;":[393],"Taplin":[394],"2000;":[397],"Schorer":[402],"Weiss,":[404],"2007).":[405],"Pairwise":[406],"akin":[409],"averaging":[411,423],"t":[414],"tests":[415],"instead":[416,427],"utilizing":[418],"an":[419,515,522,558,566],"F":[420],"test,":[421],"zero-order":[424],"correlation":[425],"employing":[429],"correlation.":[431],"problem":[433],"exacerbated":[435],"trying":[437],"define":[439],"appropriate":[440,586],"values,":[442],"as":[443],"values":[447,456],"orthogonal.":[450],"General":[451],"combining":[454],"due":[457],"Fisher":[459],"(1934,":[460],"1948)":[461],"Edgington":[463],"(1972)":[464],"require":[465],"independence":[466],"satisfied":[470],"comparisons.":[473],"Section":[475,487,503,519,528,544,563],"2,":[476],"variety":[478],"kappas":[482,580],"compared,":[486],"3":[488],"describes":[489,546],"recently":[491,532],"(Mielke":[498,539],"2007a;":[501,542],"2008),":[502,543],"4":[504,615],"discusses":[505],"procedure":[509,606],"obtain":[511,554],"observed":[516,559],"5":[520],"contains":[521],"example":[523,567,609],"analysis":[524,568],"6":[529],"introduces":[530],"published":[533],"7":[545],"8":[564],"Although":[575],"paper":[584],"any":[588],"number":[589],"r":[591],">":[592,599],"2":[593,600],"disjoint":[594],"m":[598,613],"description":[603],"confined":[611],"=":[614],"simplify":[618],"presentation.":[619],"\u2026":[620]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":3},{"year":2015,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-06-19T15:51:49.773706","created_date":"2016-06-24T00:00:00"}