{"id":"https://openalex.org/W2004141753","doi":"https://doi.org/10.1177/0962280211433597","title":"Some common errors of experimental design, interpretation and inference in agreement studies","display_name":"Some common errors of experimental design, interpretation and inference in agreement studies","publication_year":2012,"publication_date":"2012-01-09","ids":{"openalex":"https://openalex.org/W2004141753","doi":"https://doi.org/10.1177/0962280211433597","mag":"2004141753","pmid":"https://pubmed.ncbi.nlm.nih.gov/22232301"},"language":"en","primary_location":{"id":"doi:10.1177/0962280211433597","doi":"https://doi.org/10.1177/0962280211433597","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0962280211433597","pdf_url":null,"source":{"id":"https://openalex.org/S150263959","display_name":"Statistical Methods in Medical Research","issn_l":"0962-2802","issn":["0962-2802","1477-0334"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-b","anvur-class-a","anvur-scientific","cwts-core","dongbi-b","doyens","jufo-3","ki-jl-2","medline","norway-2","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Statistical Methods in Medical Research","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.rug.nl/en/publications/ac282908-e8a0-45ab-9254-1baf6cfecd81","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113770588","display_name":"Tashi P. Erdmann","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"TP Erdmann","raw_affiliation_strings":["Institute for Business and Industrial Statistics of the University of Amsterdam, Plantage Muidergracht 12, 1018 TV Amsterdam, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Business and Industrial Statistics of the University of Amsterdam, Plantage Muidergracht 12, 1018 TV Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074377651","display_name":"Jeroen de Mast","orcid":"https://orcid.org/0000-0001-8699-9799","observed_orcids":["https://orcid.org/0000-0001-8699-9799"]},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"J De Mast","raw_affiliation_strings":["Institute for Business and Industrial Statistics of the University of Amsterdam, Plantage Muidergracht 12, 1018 TV Amsterdam, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Business and Industrial Statistics of the University of Amsterdam, Plantage Muidergracht 12, 1018 TV Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029805485","display_name":"MJ Warrens","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I121797337","display_name":"Leiden University","ror":"https://ror.org/027bh9e22","country_code":"NL","type":"education","lineage":["https://openalex.org/I121797337"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"MJ Warrens","raw_affiliation_strings":["Institute of Psychology, Unit Methodology and Statistics, Leiden University, P.O. Box 9555, 2300 RB Leiden, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Psychology, Unit Methodology and Statistics, Leiden University, P.O. Box 9555, 2300 RB Leiden, The Netherlands","institution_ids":["https://openalex.org/I121797337"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":4400,"currency":"USD","value_usd":4400},"apc_paid":null,"fwci":1.8161,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.86173349,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"24","issue":"6","first_page":"920","last_page":"935"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interobserver-agreement","display_name":"interobserver agreement","score":0.7979999780654907},{"id":"https://openalex.org/keywords/kappa-index","display_name":"kappa index","score":0.9269999861717224},{"id":"https://openalex.org/keywords/methodological-errors","display_name":"methodological errors","score":0.765999972820282},{"id":"https://openalex.org/keywords/measurement-theory","display_name":"measurement theory","score":0.6460000276565552}],"concepts":[{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.6436699628829956},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6428328156471252},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.6266908049583435},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.6034419536590576},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.5819944143295288},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.5739778280258179},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.5032758116722107},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.48812541365623474},{"id":"https://openalex.org/C2777382242","wikidata":"https://www.wikidata.org/wiki/Q6017816","display_name":"Index (typography)","level":2,"score":0.458662211894989},{"id":"https://openalex.org/C164866538","wikidata":"https://www.wikidata.org/wiki/Q367351","display_name":"Cluster (spacecraft)","level":2,"score":0.42804425954818726},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42463216185569763},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4239591658115387},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4152876138687134},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38404616713523865},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18054687976837158},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D003627","descriptor_name":"Data Interpretation, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012107","descriptor_name":"Research Design","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D012494","descriptor_name":"Sampling Studies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015233","descriptor_name":"Models, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015588","descriptor_name":"Observer Variation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015982","descriptor_name":"Bias","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":7,"locations":[{"id":"doi:10.1177/0962280211433597","doi":"https://doi.org/10.1177/0962280211433597","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0962280211433597","pdf_url":null,"source":{"id":"https://openalex.org/S150263959","display_name":"Statistical Methods in Medical Research","issn_l":"0962-2802","issn":["0962-2802","1477-0334"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-b","anvur-class-a","anvur-scientific","cwts-core","dongbi-b","doyens","jufo-3","ki-jl-2","medline","norway-2","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Statistical Methods in Medical Research","raw_type":"journal-article"},{"id":"pmid:22232301","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/22232301","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Statistical methods in medical research","raw_type":"Journal Article"},{"id":"pmh:oai:pure.rug.nl:openaire/ac282908-e8a0-45ab-9254-1baf6cfecd81","doi":null,"is_oa":true,"landing_page_url":"https://research.rug.nl/en/publications/ac282908-e8a0-45ab-9254-1baf6cfecd81","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Erdmann, T P, De Mast, J & Warrens, M J 2015, 'Some common errors of experimental design, interpretation and inference in agreement studies', Statistical Methods in Medical Research, vol. 24, no. 6, pp. 920-935. https://doi.org/10.1177/0962280211433597","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:alma.31UKB_LEU:9940742034102711","doi":null,"is_oa":false,"landing_page_url":"https://scholarlypublications.universiteitleiden.nl/handle/1887/3199841","pdf_url":null,"source":{"id":"https://openalex.org/S4306400850","display_name":"Leiden Repository (Leiden University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I121797337","host_organization_name":"Leiden University","host_organization_lineage":["https://openalex.org/I121797337"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://smm.sagepub.com/content/24/6/920","raw_type":"http://purl.org/coar/resource_type/c_6501"},{"id":"pmh:oai:scholarlypublications.universiteitleiden.nl:item_3199841","doi":null,"is_oa":false,"landing_page_url":"https://hdl.handle.net/1887/3199841","pdf_url":null,"source":{"id":"https://openalex.org/S4306400850","display_name":"Leiden Repository (Leiden University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I121797337","host_organization_name":"Leiden University","host_organization_lineage":["https://openalex.org/I121797337"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article / Letter to editor"},{"id":"pmh:rug:oai:pure.rug.nl:publications/ac282908-e8a0-45ab-9254-1baf6cfecd81","doi":null,"is_oa":true,"landing_page_url":"https://research.rug.nl/en/publications/ac282908-e8a0-45ab-9254-1baf6cfecd81","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Statistical Methods in Medical Research, 24(6), 920 - 935. SAGE Publications Inc.","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:ul:oai:scholarlypublications.universiteitleiden.nl:item_3199841","doi":null,"is_oa":false,"landing_page_url":"http://hdl.handle.net/1887/3199841","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"VOLUME=24;ISSUE=6;STARTPAGE=920;ENDPAGE=935;TITLE=None","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:pure.rug.nl:openaire/ac282908-e8a0-45ab-9254-1baf6cfecd81","doi":null,"is_oa":true,"landing_page_url":"https://research.rug.nl/en/publications/ac282908-e8a0-45ab-9254-1baf6cfecd81","pdf_url":null,"source":{"id":"https://openalex.org/S4306400420","display_name":"University of Groningen research database (University of Groningen / Centre for Information Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169381384","host_organization_name":"University of Groningen","host_organization_lineage":["https://openalex.org/I169381384"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Erdmann, T P, De Mast, J & Warrens, M J 2015, 'Some common errors of experimental design, interpretation and inference in agreement studies', Statistical Methods in Medical Research, vol. 24, no. 6, pp. 920-935. https://doi.org/10.1177/0962280211433597","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W1522545672","https://openalex.org/W1539462193","https://openalex.org/W1547900692","https://openalex.org/W1586414640","https://openalex.org/W1966184253","https://openalex.org/W1972586296","https://openalex.org/W1975879668","https://openalex.org/W1978217317","https://openalex.org/W1979773093","https://openalex.org/W1983897914","https://openalex.org/W1984157647","https://openalex.org/W1985575436","https://openalex.org/W1986523461","https://openalex.org/W1997449248","https://openalex.org/W2031738304","https://openalex.org/W2032087855","https://openalex.org/W2040447587","https://openalex.org/W2040717102","https://openalex.org/W2053154970","https://openalex.org/W2055008194","https://openalex.org/W2067978628","https://openalex.org/W2070285512","https://openalex.org/W2076061140","https://openalex.org/W2077906074","https://openalex.org/W2082415722","https://openalex.org/W2084398551","https://openalex.org/W2102863441","https://openalex.org/W2111877222","https://openalex.org/W2130108777","https://openalex.org/W2144489542","https://openalex.org/W2149490731","https://openalex.org/W2149584926","https://openalex.org/W2150290224","https://openalex.org/W2153942520","https://openalex.org/W2157318188","https://openalex.org/W2158283933","https://openalex.org/W2164777277","https://openalex.org/W2315214008","https://openalex.org/W2626116100","https://openalex.org/W4246593158","https://openalex.org/W4388317518"],"related_works":["https://openalex.org/W7096838685","https://openalex.org/W7098961846","https://openalex.org/W2070285512","https://openalex.org/W7155753529","https://openalex.org/W2158283933","https://openalex.org/W1960172443","https://openalex.org/W2142147467","https://openalex.org/W1444168786","https://openalex.org/W2153072998","https://openalex.org/W1540240814"],"abstract_inverted_index":{"We":[0],"signal":[1],"and":[2,10,23,46,172],"discuss":[3],"common":[4],"methodological":[5],"errors":[6,53,95,125,154],"in":[7,18,20,37,44,61,66,91,106,161,167,187],"agreement":[8,137,175],"studies":[9,159],"the":[11,21,40,67,77,80,97,101,109,112,117,145,148,178,184],"use":[12,78,181],"of":[13,52,79,96,111,124,147,170],"kappa":[14,81,185],"indices,":[15],"as":[16,85],"found":[17],"publications":[19],"medical":[22],"behavioural":[24],"sciences.":[25],"Our":[26],"analysis":[27,165],"is":[28,36,54],"based":[29,135],"on":[30,136,144],"a":[31,62,168],"proposed":[32,149],"statistical":[33,150],"model":[34],"that":[35],"line":[38],"with":[39],"typical":[41],"models":[42],"employed":[43],"metrology":[45],"measurement":[47],"theory.":[48],"A":[49,121],"first":[50],"cluster":[51,123],"related":[55],"to":[56,132,180,183],"nonrandom":[57],"sampling,":[58],"which":[59,129],"results":[60,166],"potentially":[63],"substantial":[64],"bias":[65],"estimated":[68],"agreement.":[69],"Second,":[70],"when":[71],"class":[72,118],"prevalences":[73,119],"are":[74,142,155],"strongly":[75],"nonuniform,":[76],"index":[82,102,186],"becomes":[83],"precarious,":[84],"its":[86],"large":[87,93],"partial":[88],"derivatives":[89],"result":[90],"typically":[92],"standard":[94],"estimates.":[98],"In":[99],"addition,":[100],"reflects":[103],"rather":[104],"one-sidedly":[105],"such":[107],"cases":[108],"consistency":[110],"most":[113],"prevalent":[114],"class,":[115],"or":[116],"themselves.":[120],"final":[122],"concerns":[126],"interpretation":[127,140],"pitfalls,":[128],"may":[130],"lead":[131],"incorrect":[133],"conclusions":[134],"studies.":[138],"These":[139],"issues":[141],"clarified":[143],"basis":[146],"modelling.":[151],"The":[152,164],"signalled":[153],"illustrated":[156],"from":[157],"actual":[158],"published":[160],"prestigious":[162],"journals.":[163],"number":[169],"guidelines":[171],"recommendations":[173],"for":[174],"studies,":[176],"including":[177],"recommendation":[179],"alternatives":[182],"certain":[188],"situations.":[189]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":1}],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2016-06-24T00:00:00"}