{"id":"https://openalex.org/W2987558058","doi":"https://doi.org/10.1109/access.2019.2953104","title":"A Simplified Cohen\u2019s Kappa for Use in Binary Classification Data Annotation Tasks","display_name":"A Simplified Cohen\u2019s Kappa for Use in Binary Classification Data Annotation Tasks","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2987558058","doi":"https://doi.org/10.1109/access.2019.2953104","mag":"2987558058"},"language":"en","primary_location":{"id":"doi:10.1109/access.2019.2953104","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2953104","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08896957.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08896957.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100347484","display_name":"Juan Wang","orcid":"https://orcid.org/0000-0003-3124-9901"},"institutions":[{"id":"https://openalex.org/I1320351128","display_name":"Delta Air Lines (United States)","ror":"https://ror.org/03g9c1e75","country_code":"US","type":"company","lineage":["https://openalex.org/I1320351128"]},{"id":"https://openalex.org/I4210139151","display_name":"Innovative Micro Technology (United States)","ror":"https://ror.org/03w6g4z68","country_code":"US","type":"company","lineage":["https://openalex.org/I4210139151"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juan Wang","raw_affiliation_strings":["Delta Micro Technology Inc., Laguna Hills, USA"],"raw_orcid":"https://orcid.org/0000-0003-3124-9901","affiliations":[{"raw_affiliation_string":"Delta Micro Technology Inc., Laguna Hills, USA","institution_ids":["https://openalex.org/I4210139151","https://openalex.org/I1320351128"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048510347","display_name":"Yongyi Yang","orcid":"https://orcid.org/0000-0003-3564-3260"},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yongyi Yang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Illinois Institute of Technology, Chicago, USA"],"raw_orcid":"https://orcid.org/0000-0003-3564-3260","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Illinois Institute of Technology, Chicago, USA","institution_ids":["https://openalex.org/I180949307"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028343047","display_name":"Bin Xia","orcid":"https://orcid.org/0000-0002-0340-8082"},"institutions":[{"id":"https://openalex.org/I4210164792","display_name":"Everbright International (China)","ror":"https://ror.org/05tedyt08","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210164792"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Xia","raw_affiliation_strings":["Shenzhen SiBright Company Ltd., Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-0340-8082","affiliations":[{"raw_affiliation_string":"Shenzhen SiBright Company Ltd., Shenzhen, China","institution_ids":["https://openalex.org/I4210164792"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":2.582,"has_fulltext":true,"cited_by_count":52,"citation_normalized_percentile":{"value":0.92431738,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"7","issue":null,"first_page":"164386","last_page":"164397"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cohens-kappa","display_name":"Cohen's kappa","score":0.7137163877487183},{"id":"https://openalex.org/keywords/kappa","display_name":"Kappa","score":0.7091865539550781},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6043250560760498},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5846716165542603},{"id":"https://openalex.org/keywords/binary-classification","display_name":"Binary classification","score":0.5758025646209717},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5146423578262329},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.49189186096191406},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.4892735183238983},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4758163094520569},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.4711631238460541},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.43490397930145264},{"id":"https://openalex.org/keywords/statistic","display_name":"Statistic","score":0.43392759561538696},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.42049363255500793},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.40185078978538513},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.39405179023742676},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3241390883922577},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.27567917108535767},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"Support vector machine","score":0.10372641682624817}],"concepts":[{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.7137163877487183},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.7091865539550781},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6043250560760498},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5846716165542603},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.5758025646209717},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5146423578262329},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.49189186096191406},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.4892735183238983},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4758163094520569},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.4711631238460541},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.43490397930145264},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.43392759561538696},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.42049363255500793},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.40185078978538513},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.39405179023742676},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3241390883922577},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27567917108535767},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.10372641682624817},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2019.2953104","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2953104","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08896957.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:38da67f28e2341f7b2db19a397035a44","is_oa":true,"landing_page_url":"https://doaj.org/article/38da67f28e2341f7b2db19a397035a44","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 7, Pp 164386-164397 (2019)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2019.2953104","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2953104","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08896957.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4000000059604645,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2987558058.pdf","grobid_xml":"https://content.openalex.org/works/W2987558058.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1444168786","https://openalex.org/W1860567973","https://openalex.org/W1966184253","https://openalex.org/W1975128126","https://openalex.org/W2000258708","https://openalex.org/W2046280714","https://openalex.org/W2053154970","https://openalex.org/W2055472916","https://openalex.org/W2070285512","https://openalex.org/W2089349281","https://openalex.org/W2105897558","https://openalex.org/W2125591592","https://openalex.org/W2143702645","https://openalex.org/W2152575748","https://openalex.org/W2153072998","https://openalex.org/W2153804780","https://openalex.org/W2154178322","https://openalex.org/W2157617842","https://openalex.org/W2163037194","https://openalex.org/W2167460663","https://openalex.org/W2170021941","https://openalex.org/W2170645625","https://openalex.org/W2295951612","https://openalex.org/W2524399695","https://openalex.org/W2557738935","https://openalex.org/W2575657035","https://openalex.org/W2581082771","https://openalex.org/W2589453516","https://openalex.org/W2623511780","https://openalex.org/W2783597708","https://openalex.org/W2966300704","https://openalex.org/W6628505203","https://openalex.org/W6677082149","https://openalex.org/W6682780409","https://openalex.org/W6683674869"],"related_works":["https://openalex.org/W2013743689","https://openalex.org/W1540240814","https://openalex.org/W2352955518","https://openalex.org/W3121132670","https://openalex.org/W1490963911","https://openalex.org/W1559017864","https://openalex.org/W2017448678","https://openalex.org/W1972248331","https://openalex.org/W3125983537","https://openalex.org/W2007212974"],"abstract_inverted_index":{"In":[0,68],"binary":[1,110],"classification":[2,36,55,111],"tasks,":[3],"Cohen's":[4,84],"kappa":[5,32,63,177,199],"is":[6,17,42,46,97,148,155,181,203,217],"often":[7],"used":[8,35],"as":[9,23],"a":[10,79,106,131,141,157,191],"quality":[11],"measure":[12,209],"for":[13,51,83,109,210],"data":[14],"annotations,":[15],"which":[16],"inconsistent":[18],"with":[19],"its":[20],"original":[21],"purpose":[22],"an":[24,73,115,184,207],"inter-annotator":[25],"consistency":[26],"measure.":[27],"The":[28,150,196],"analytic":[29],"relationship":[30,82,96,124,138,164,175],"between":[31,176,198],"and":[33,40,44,87,125,129,169,178,200],"commonly":[34],"metrics":[37],"(e.g.,":[38],"sensitivity":[39],"specificity)":[41],"nonlinear,":[43],"thus":[45],"difficult":[47],"to":[48,102,135,205],"be":[49,206],"applied":[50],"interpretation":[52],"of":[53,61,65],"the":[54,59,62,66,91,118,122,126,137,145,162,174],"performance":[56,107],"(merely":[57],"from":[58,187],"knowledge":[60],"value)":[64],"annotations.":[67],"this":[69],"study,":[70],"based":[71],"on":[72,117,183],"annotation":[74,185],"generation":[75],"model,":[76],"we":[77],"derive":[78],"simplified,":[80],"linear":[81,119,132],"kappa,":[85],"sensitivity,":[86],"specificity":[88],"by":[89,100,139],"using":[90,140],"1st-order":[92],"Taylor":[93],"approximation.":[94],"This":[95],"further":[98],"simplified":[99,123,163],"relating":[101],"Youden's":[103,179,201],"J":[104,180,202],"statistic,":[105],"metric":[108],"tasks.":[112],"We":[113],"provide":[114],"analysis":[116,134],"coefficients":[120],"in":[121,161,190],"approximation":[127,159],"error,":[128],"conduct":[130],"regression":[133],"assess":[136],"synthetic":[142],"dataset":[143,186],"where":[144],"ground":[146,215],"truth":[147,216],"known.":[149],"results":[151],"show":[152],"that":[153],"there":[154],"only":[156],"negligible":[158],"error":[160],"when":[165,213],"no":[166,214],"major":[167],"bias":[168],"prevalence":[170],"issues":[171],"exist.":[172],"Furthermore,":[173],"validated":[182],"seven":[188],"graders":[189],"diabetic":[192],"retinopathy":[193],"screening":[194],"study.":[195],"discrepancy":[197],"demonstrated":[204],"effective":[208],"annotator":[211],"assessment":[212],"available.":[218]},"counts_by_year":[{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}