{"id":"https://openalex.org/W4224989672","doi":"https://doi.org/10.3390/sym14020262","title":"An Empirical Comparative Assessment of Inter-Rater Agreement of Binary Outcomes and Multiple Raters","display_name":"An Empirical Comparative Assessment of Inter-Rater Agreement of Binary Outcomes and Multiple Raters","publication_year":2022,"publication_date":"2022-01-29","ids":{"openalex":"https://openalex.org/W4224989672","doi":"https://doi.org/10.3390/sym14020262"},"language":"en","primary_location":{"id":"doi:10.3390/sym14020262","is_oa":true,"landing_page_url":"https://doi.org/10.3390/sym14020262","pdf_url":"https://www.mdpi.com/2073-8994/14/2/262/pdf?version=1643437853","source":{"id":"https://openalex.org/S190787756","display_name":"Symmetry","issn_l":"2073-8994","issn":["2073-8994"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Symmetry","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2073-8994/14/2/262/pdf?version=1643437853","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008176056","display_name":"Menelaos Konstantinidis","orcid":"https://orcid.org/0000-0002-5074-9023"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210125326","display_name":"Public Health Ontario","ror":"https://ror.org/025z8ah66","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I4210125326"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Menelaos Konstantinidis","raw_affiliation_strings":["Division of Biostatistics, Dalla Lana School of Public Health, University of Toronto, Toronto, ON M5T 3M7, Canada"],"affiliations":[{"raw_affiliation_string":"Division of Biostatistics, Dalla Lana School of Public Health, University of Toronto, Toronto, ON M5T 3M7, Canada","institution_ids":["https://openalex.org/I4210125326","https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071526496","display_name":"Lisa W. Le","orcid":"https://orcid.org/0000-0002-0084-9915"},"institutions":[{"id":"https://openalex.org/I1325899441","display_name":"University Health Network","ror":"https://ror.org/042xt5161","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I1325899441"]},{"id":"https://openalex.org/I2802654988","display_name":"Princess Margaret Cancer Centre","ror":"https://ror.org/03zayce58","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I1325899441","https://openalex.org/I2802654988"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Lisa. W. Le","raw_affiliation_strings":["Department of Biostatistics, Princess Margaret Cancer Centre, University Health Network, Toronto, ON M5G 2C1, Canada"],"affiliations":[{"raw_affiliation_string":"Department of Biostatistics, Princess Margaret Cancer Centre, University Health Network, Toronto, ON M5G 2C1, Canada","institution_ids":["https://openalex.org/I2802654988","https://openalex.org/I1325899441"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004008569","display_name":"Xin Gao","orcid":"https://orcid.org/0000-0001-8057-5100"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Xin Gao","raw_affiliation_strings":["Department of Mathematics and Statistics, York University, Toronto, ON M3J 1P3, Canada"],"affiliations":[{"raw_affiliation_string":"Department of Mathematics and Statistics, York University, Toronto, ON M3J 1P3, Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5004008569","https://openalex.org/A5071526496"],"corresponding_institution_ids":["https://openalex.org/I1325899441","https://openalex.org/I192455969","https://openalex.org/I2802654988"],"apc_list":{"value":2000,"currency":"CHF","value_usd":2165},"apc_paid":{"value":2000,"currency":"CHF","value_usd":2165},"fwci":4.2846,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.94940955,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"14","issue":"2","first_page":"262","last_page":"262"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.817299485206604},{"id":"https://openalex.org/keywords/cohens-kappa","display_name":"Cohen's kappa","score":0.7895157337188721},{"id":"https://openalex.org/keywords/kappa","display_name":"Kappa","score":0.7170439958572388},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"Inter-rater reliability","score":0.6156550645828247},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5972161293029785},{"id":"https://openalex.org/keywords/statistic","display_name":"Statistic","score":0.5292707681655884},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.48339763283729553},{"id":"https://openalex.org/keywords/summary-statistics","display_name":"Summary statistics","score":0.4632519483566284},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.42874905467033386},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.22052928805351257}],"concepts":[{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.817299485206604},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.7895157337188721},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.7170439958572388},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6156550645828247},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5972161293029785},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.5292707681655884},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.48339763283729553},{"id":"https://openalex.org/C32792767","wikidata":"https://www.wikidata.org/wiki/Q1370484","display_name":"Summary statistics","level":2,"score":0.4632519483566284},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.42874905467033386},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.22052928805351257},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.0},{"id":"https://openalex.org/C99454951","wikidata":"https://www.wikidata.org/wiki/Q932068","display_name":"Environmental health","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.3390/sym14020262","is_oa":true,"landing_page_url":"https://doi.org/10.3390/sym14020262","pdf_url":"https://www.mdpi.com/2073-8994/14/2/262/pdf?version=1643437853","source":{"id":"https://openalex.org/S190787756","display_name":"Symmetry","issn_l":"2073-8994","issn":["2073-8994"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Symmetry","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:6a3fc03918734e2d919d7f123b42ec06","is_oa":true,"landing_page_url":"https://doaj.org/article/6a3fc03918734e2d919d7f123b42ec06","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Symmetry, Vol 14, Iss 2, p 262 (2022)","raw_type":"article"},{"id":"pmh:oai:mdpi.com:/2073-8994/14/2/262/","is_oa":true,"landing_page_url":"https://dx.doi.org/10.3390/sym14020262","pdf_url":null,"source":{"id":"https://openalex.org/S4306400947","display_name":"MDPI (MDPI AG)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210097602","host_organization_name":"Multidisciplinary Digital Publishing Institute (Switzerland)","host_organization_lineage":["https://openalex.org/I4210097602"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Symmetry","raw_type":"Text"},{"id":"pmh:oai:tspace.library.utoronto.ca:1807/110131","is_oa":true,"landing_page_url":"http://hdl.handle.net/1807/110131","pdf_url":null,"source":{"id":"https://openalex.org/S4306400261","display_name":"TSpace (University of Toronto)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I185261750","host_organization_name":"University of Toronto","host_organization_lineage":["https://openalex.org/I185261750"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":{"id":"doi:10.3390/sym14020262","is_oa":true,"landing_page_url":"https://doi.org/10.3390/sym14020262","pdf_url":"https://www.mdpi.com/2073-8994/14/2/262/pdf?version=1643437853","source":{"id":"https://openalex.org/S190787756","display_name":"Symmetry","issn_l":"2073-8994","issn":["2073-8994"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Symmetry","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6099133885","display_name":null,"funder_award_id":"123456 288332","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4224989672.pdf","grobid_xml":"https://content.openalex.org/works/W4224989672.grobid-xml"},"referenced_works_count":53,"referenced_works":["https://openalex.org/W1526057606","https://openalex.org/W1674084917","https://openalex.org/W1975879668","https://openalex.org/W1976658129","https://openalex.org/W1978217317","https://openalex.org/W1982264966","https://openalex.org/W1989492805","https://openalex.org/W1995422727","https://openalex.org/W1996290308","https://openalex.org/W1996445490","https://openalex.org/W2005287859","https://openalex.org/W2015795623","https://openalex.org/W2019029324","https://openalex.org/W2023549687","https://openalex.org/W2023701460","https://openalex.org/W2026856125","https://openalex.org/W2029553347","https://openalex.org/W2037789405","https://openalex.org/W2051779656","https://openalex.org/W2052005998","https://openalex.org/W2053154970","https://openalex.org/W2063743914","https://openalex.org/W2069378996","https://openalex.org/W2070285512","https://openalex.org/W2071842481","https://openalex.org/W2076898331","https://openalex.org/W2096721022","https://openalex.org/W2106989261","https://openalex.org/W2107328434","https://openalex.org/W2115218353","https://openalex.org/W2127832821","https://openalex.org/W2130977084","https://openalex.org/W2131915193","https://openalex.org/W2164907862","https://openalex.org/W2172216441","https://openalex.org/W2182269197","https://openalex.org/W2477907264","https://openalex.org/W2531269403","https://openalex.org/W2567362164","https://openalex.org/W2734668096","https://openalex.org/W2743746188","https://openalex.org/W2758667827","https://openalex.org/W2767975651","https://openalex.org/W2773350519","https://openalex.org/W2790690319","https://openalex.org/W2792454816","https://openalex.org/W2893300601","https://openalex.org/W2901669506","https://openalex.org/W2902193528","https://openalex.org/W2970684805","https://openalex.org/W3000424167","https://openalex.org/W3012624299","https://openalex.org/W4254697198"],"related_works":["https://openalex.org/W1966976587","https://openalex.org/W4294214983","https://openalex.org/W2054015915","https://openalex.org/W1984207165","https://openalex.org/W1979852909","https://openalex.org/W96748260","https://openalex.org/W2581818947","https://openalex.org/W1976976697","https://openalex.org/W2330902540","https://openalex.org/W1996751524"],"abstract_inverted_index":{"Background:":[0],"Many":[1],"methods":[2],"under":[3],"the":[4,40,51,73,96,103,114,118,123,130,141,146,157,166,187],"umbrella":[5],"of":[6,26,30,42,53,67,78,98,107,122,169,172],"inter-rater":[7,82],"agreement":[8,83],"(IRA)":[9],"have":[10],"been":[11],"proposed":[12],"to":[13,34],"evaluate":[14],"how":[15],"well":[16],"two":[17],"or":[18],"more":[19],"medical":[20],"experts":[21],"agree":[22],"on":[23],"a":[24,136,152,170,178],"set":[25,171],"outcomes.":[27,47],"The":[28],"objective":[29],"this":[31],"work":[32],"was":[33],"assess":[35],"key":[36],"IRA":[37,196],"statistics":[38,84,110,126,160,200],"in":[39],"context":[41],"multiple":[43],"raters":[44,55,68],"with":[45,57],"binary":[46],"Methods:":[48],"We":[49],"simulated":[50],"responses":[52],"several":[54],"(2\u20135)":[56],"20,":[58],"50,":[59],"300,":[60],"and":[61,69,76,91],"500":[62],"observations.":[63],"For":[64,191],"each":[65,162],"combination":[66],"observations,":[70],"we":[71],"estimated":[72,104,119,138,154],"expected":[74,105,120],"value":[75],"variance":[77,139,155],"four":[79,109],"commonly":[80],"used":[81],"(Fleiss\u2019":[85],"Kappa,":[86,88,90],"Light\u2019s":[87],"Conger\u2019s":[89],"Gwet\u2019s":[92,149,180],"AC1).":[93],"Results:":[94],"In":[95,113,129,145],"case":[97],"equal":[99],"outcome":[100],"prevalence":[101,168],"(symmetric),":[102],"values":[106,121],"all":[108],"were":[111,127],"equal.":[112,128],"asymmetric":[115,147],"case,":[116,132,148],"only":[117],"three":[124,143,158,188],"Kappa":[125,134,159,189],"symmetric":[131],"Fleiss\u2019":[133],"yielded":[135,151],"higher":[137],"than":[140,156],"other":[142],"statistics.":[144,190],"AC1":[150,181],"lower":[153],"for":[161],"scenario.":[163],"Conclusion:":[164],"Since":[165],"population-level":[167],"outcomes":[173],"may":[174],"not":[175],"be":[176,184,202],"known":[177],"priori,":[179],"statistic":[182],"should":[183,201],"favored":[185],"over":[186],"meaningful":[192],"direct":[193],"comparisons":[194],"between":[195,199],"measures,":[197],"transformations":[198],"conducted.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":4}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-10-10T00:00:00"}