{"id":"https://openalex.org/W7225323998","doi":"https://doi.org/10.31234/osf.io/2ctua","title":"Fixed Raters, Random Selection: A Correlation-Based Framework for Agreement ICCs in Two-Way Mixed-Effects Designs","display_name":"Fixed Raters, Random Selection: A Correlation-Based Framework for Agreement ICCs in Two-Way Mixed-Effects Designs","publication_year":2026,"publication_date":"2026-11-05","ids":{"openalex":"https://openalex.org/W7225323998","doi":"https://doi.org/10.31234/osf.io/2ctua"},"language":"en","primary_location":{"id":"doi:10.31234/osf.io/2ctua","doi":"https://doi.org/10.31234/osf.io/2ctua","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/2ctua","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.31234/osf.io/2ctua","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076972842","display_name":"Tzu\u2010Yao Lin","orcid":"https://orcid.org/0000-0002-2261-448X","observed_orcids":["https://orcid.org/0000-0002-2261-448X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tzu-Yao Lin","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-2261-448X","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045014848","display_name":"Francis Tuerlinckx","orcid":"https://orcid.org/0000-0002-1775-7654","observed_orcids":["https://orcid.org/0000-0002-1775-7654"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"francis tuerlinckx","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-1775-7654","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5048679258","display_name":"Sophie Vanbelle","orcid":"https://orcid.org/0000-0001-6584-2522","observed_orcids":["https://orcid.org/0000-0001-6584-2522"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sophie Vanbelle","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-6584-2522","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10243","display_name":"Statistical Methods and Bayesian Inference","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/intraclass-correlation-coefficient","display_name":"intraclass correlation coefficient","score":0.7879999876022339},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.9240000247955322},{"id":"https://openalex.org/keywords/finite-population-sampling","display_name":"finite population sampling","score":0.8309999704360962},{"id":"https://openalex.org/keywords/measurement-reliability","display_name":"measurement reliability","score":0.49399998784065247}],"concepts":[{"id":"https://openalex.org/C104709138","wikidata":"https://www.wikidata.org/wiki/Q1671540","display_name":"Intraclass correlation","level":3,"score":0.7799999713897705},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.7552000284194946},{"id":"https://openalex.org/C2779545769","wikidata":"https://www.wikidata.org/wiki/Q5135364","display_name":"Closeness","level":2,"score":0.6545000076293945},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5939000248908997},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5598000288009644},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5537999868392944},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5321999788284302},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.516700029373169},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.49570000171661377},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.4749999940395355},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4339999854564667},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.37619999051094055},{"id":"https://openalex.org/C69941244","wikidata":"https://www.wikidata.org/wiki/Q6045770","display_name":"Interclass correlation","level":4,"score":0.3695000112056732},{"id":"https://openalex.org/C103423394","wikidata":"https://www.wikidata.org/wiki/Q5172857","display_name":"Correlation ratio","level":3,"score":0.3458999991416931},{"id":"https://openalex.org/C168743327","wikidata":"https://www.wikidata.org/wiki/Q1826427","display_name":"Random effects model","level":3,"score":0.3440999984741211},{"id":"https://openalex.org/C177562468","wikidata":"https://www.wikidata.org/wiki/Q182893","display_name":"Dispersion (optics)","level":2,"score":0.3400999903678894},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3158000111579895},{"id":"https://openalex.org/C17231256","wikidata":"https://www.wikidata.org/wiki/Q5156540","display_name":"Completeness (order theory)","level":2,"score":0.31290000677108765},{"id":"https://openalex.org/C2780092901","wikidata":"https://www.wikidata.org/wiki/Q3433612","display_name":"Correlation coefficient","level":2,"score":0.2969000041484833},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.29280000925064087},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2888999879360199},{"id":"https://openalex.org/C106195933","wikidata":"https://www.wikidata.org/wiki/Q7847935","display_name":"Truncation (statistics)","level":2,"score":0.2858000099658966},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.28200000524520874},{"id":"https://openalex.org/C122123141","wikidata":"https://www.wikidata.org/wiki/Q176623","display_name":"Random variable","level":2,"score":0.2786000072956085},{"id":"https://openalex.org/C55078378","wikidata":"https://www.wikidata.org/wiki/Q1136628","display_name":"Pearson product-moment correlation coefficient","level":2,"score":0.26579999923706055},{"id":"https://openalex.org/C55786151","wikidata":"https://www.wikidata.org/wiki/Q951437","display_name":"Level of measurement","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.2630000114440918},{"id":"https://openalex.org/C12426560","wikidata":"https://www.wikidata.org/wiki/Q189569","display_name":"Basis (linear algebra)","level":2,"score":0.2533000111579895},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.2517000138759613}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.31234/osf.io/2ctua","doi":"https://doi.org/10.31234/osf.io/2ctua","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/2ctua","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},{"id":"pmh:oai:share.osf.io:238498f7-fc3b-4db6-954c-2bab72cfd37c","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/2ctua","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.31234/osf.io/2ctua","doi":"https://doi.org/10.31234/osf.io/2ctua","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/2ctua","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4294123474","https://openalex.org/W1991855103","https://openalex.org/W7095963081","https://openalex.org/W2416643937","https://openalex.org/W2913497051","https://openalex.org/W2288200086","https://openalex.org/W2038275409","https://openalex.org/W4402557897","https://openalex.org/W2240403635","https://openalex.org/W7130927241"],"abstract_inverted_index":{"Intraclass":[0],"correlation":[1,112],"coefficients":[2],"(ICCs)":[3],"quantify":[4],"the":[5,11,27,36,74,83,94,111,139,160,197],"reliability":[6,19],"of":[7,30,38,71,148],"measurements":[8],"made":[9],"on":[10,196],"same":[12],"persons":[13],"by":[14,103],"different":[15],"raters.":[16],"In":[17,48,96],"interrater":[18],"studies,":[20],"consistency":[21],"ICCs":[22,34,60,124,150],"assess":[23],"whether":[24],"scores":[25,40],"preserve":[26],"relative":[28],"rankings":[29],"persons,":[31],"while":[32],"agreement":[33,59,123,149,162,181],"evaluate":[35],"closeness":[37],"absolute":[39],"across":[41,191],"raters,":[42],"thereby":[43],"penalising":[44],"systematic":[45],"rater":[46,75],"differences.":[47],"two-way":[49],"mixed-effects":[50,152],"designs,":[51],"where":[52],"raters":[53,186],"are":[54,187],"treated":[55],"as":[56],"fixed,":[57],"conventional":[58,161],"raise":[61],"a":[62,69,105,133,146],"conceptual":[63],"paradox:":[64],"fixed-rater":[65],"dispersion":[66],"contributes":[67],"to":[68],"source":[70],"disagreement":[72],"despite":[73],"effects":[76,91],"being":[77],"constants,":[78],"and":[79,129,174,190],"it":[80],"remains":[81],"in":[82,93,110,151],"coefficient":[84],"for":[85,125],"all-rater":[86,130],"averages":[87],"even":[88],"though":[89],"those":[90],"cancel":[92],"average.":[95],"this":[97,101,118],"paper,":[98],"we":[99,120],"resolve":[100],"paradox":[102],"introducing":[104],"finite-population":[106],"sampling":[107,169],"framework":[108],"grounded":[109],"approach":[113],"with":[114,170],"hypothetical":[115],"replications.":[116],"Within":[117],"framework,":[119],"derive":[121],"correlation-based":[122],"both":[126,172],"single":[127],"ratings":[128],"averages,":[131],"establishing":[132],"coherent":[134],"probabilistic":[135],"foundation":[136],"that":[137,159],"resolves":[138],"paradox.":[140],"By":[141],"considering":[142],"various":[143],"rater-sampling":[144],"schemes,":[145],"family":[147],"designs":[153],"is":[154],"presented.":[155],"We":[156],"further":[157],"show":[158],"formulas":[163],"can":[164],"be":[165],"recovered":[166],"under":[167],"independent":[168],"replacement":[171],"within":[173,189],"between":[175],"replicates.":[176],"Thus,":[177],"choosing":[178],"an":[179],"appropriate":[180],"ICC":[182],"requires":[183],"specifying":[184],"how":[185],"sampled":[188],"replicates,":[192],"rather":[193],"than":[194],"relying":[195],"design":[198],"alone.":[199]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-11-06T00:00:00"}