{"id":"https://openalex.org/W7095209883","doi":null,"title":"Benchmarking rwg interrater agreement indices: Let\u2019s drop the .70 ruleof-thumb. Paper presented at","display_name":"Benchmarking rwg interrater agreement indices: Let\u2019s drop the .70 ruleof-thumb. Paper presented at","publication_year":2004,"publication_date":"2004-01-01","ids":{"openalex":"https://openalex.org/W7095209883"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.553.6887","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.553.6887","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://harvey.psyc.vt.edu/Documents/2004SIOP_harvey_hollander.pdf","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Robert J. Harvey","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Robert J. Harvey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Eran Holl","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eran Holl","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.51020408,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.0017000000225380063,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12443","display_name":"Delphi Technique in Research","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interobserver-agreement","display_name":"interobserver agreement","score":0.902999997138977},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.39399999380111694},{"id":"https://openalex.org/keywords/monte-carlo-simulation","display_name":"Monte Carlo simulation","score":0.5860000252723694}],"concepts":[{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.8252999782562256},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8159000277519226},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.6258000135421753},{"id":"https://openalex.org/C3018577593","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Limits of agreement","level":2,"score":0.4187000095844269},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.41760000586509705},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.40869998931884766},{"id":"https://openalex.org/C2984949393","wikidata":"https://www.wikidata.org/wiki/Q1260241","display_name":"Drop out","level":2,"score":0.3508000075817108},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.349700003862381},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.32580000162124634},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.304500013589859},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29910001158714294},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2856999933719635},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.2687999904155731},{"id":"https://openalex.org/C2986587452","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical analysis","level":2,"score":0.2590000033378601}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.553.6887","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.553.6887","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://harvey.psyc.vt.edu/Documents/2004SIOP_harvey_hollander.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W1971296148","https://openalex.org/W2054658141","https://openalex.org/W1963965233","https://openalex.org/W2022950201","https://openalex.org/W2135758045","https://openalex.org/W2059394167","https://openalex.org/W2077352251","https://openalex.org/W2144772627","https://openalex.org/W1978684715","https://openalex.org/W7097398150"],"abstract_inverted_index":{"Variance-based":[0],"interrater":[1,59],"agreement":[2,63],"indices":[3,51,86],"in":[4],"the":[5,88,93],"rWG":[6,83],"family":[7,84],"are":[8,29],"often":[9],"interpreted":[10],"using":[11],"rules-of-thumb":[12],"derived":[13],"for":[14,32],"reliabilities":[15],"(e.g.,":[16,57,64,95,105],"\u2265.70":[17],"=":[18],"acceptable).":[19],"Monte":[20],"Carlo":[21],"results":[22],"suggest":[23],"that":[24],"far":[25],"more":[26],"stringent":[27],"standards":[28],"needed,":[30],"especially":[31],"maximum-variance":[33],"rWG,":[34],"as":[35],"values>.70":[36],"can":[37],"routinely":[38],"be":[39],"obtained":[40],"from":[41,92],"totally":[42],"random":[43],"ratings.":[44],"Researchers":[45],"have":[46],"developed":[47],"a":[48],"range":[49],"of":[50,85],"sensitive":[52],"to":[53,101],"pattern-based":[54],"ratings":[55],"similarity":[56],"average":[58],"r)":[60],"and/or":[61],"level-based":[62],"see":[65],"Dunlap,":[66],"Burke,":[67],"&":[68,74,78,98],"Smith-Crowe,":[69],"2003,":[70],"Mitchell,":[71],"1979;":[72,76],"Shrout":[73],"Fleiss,":[75],"Tinsley":[77],"Weiss,":[79],"1975).":[80],"The":[81],"popular":[82],"takes":[87],"latter":[89],"approach,":[90],"evolving":[91],"original":[94],"James,":[96],"Demaree,":[97],"Wolf,":[99],"1984)":[100],"include":[102],"several":[103],"derivatives":[104],"Brown,":[106],"2002;":[107]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-27T00:00:00"}