{"id":"https://openalex.org/W2804034219","doi":null,"title":"An analysis of rater effects in reviews of scientific manuscripts","display_name":"An analysis of rater effects in reviews of scientific manuscripts","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2804034219","mag":"2804034219"},"language":"en","primary_location":{"id":"pmh:oai:libres.uncg.edu/22620","doi":null,"is_oa":false,"landing_page_url":"http://libres.uncg.edu/ir/uncg/f/Turner_uncg_0154D_12351.pdf","pdf_url":"http://libres.uncg.edu/ir/uncg/f/Turner_uncg_0154D_12351.pdf","source":{"id":"https://openalex.org/S4306401856","display_name":"NC Digital Online Collection of Knowledge and Scholarship (The University of North Carolina at Greensboro)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169335092","host_organization_name":"University of North Carolina at Greensboro","host_organization_lineage":["https://openalex.org/I169335092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008683504","display_name":"Dana P. Turner","orcid":"https://orcid.org/0000-0001-8573-3550","observed_orcids":["https://orcid.org/0000-0001-8573-3550"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Dana P. Turner","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5008683504"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.7329000234603882,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.7329000234603882,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13284","display_name":"Psychology Research and Bibliometrics","score":0.0820000022649765,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.03139999881386757,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-review","display_name":"peer review","score":0.878000020980835},{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.765999972820282},{"id":"https://openalex.org/keywords/publication-decisions","display_name":"publication decisions","score":0.7710000276565552},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"generalizability theory","score":0.8939999938011169},{"id":"https://openalex.org/keywords/many-facet-rasch-measurement","display_name":"many-facet Rasch model","score":0.9919999837875366},{"id":"https://openalex.org/keywords/decision-studies","display_name":"decision studies","score":0.703000009059906},{"id":"https://openalex.org/keywords/reliability-statistics","display_name":"reliability statistics","score":0.35199999809265137},{"id":"https://openalex.org/keywords/scholarly-journals","display_name":"scientific journals","score":0.5950000286102295},{"id":"https://openalex.org/keywords/reviewer-training","display_name":"reviewer training","score":0.7649999856948853},{"id":"https://openalex.org/keywords/generalizability-coefficient","display_name":"generalizability coefficient","score":0.5849999785423279},{"id":"https://openalex.org/keywords/research-ethics-education","display_name":"research ethics training","score":0.32100000977516174}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8709344863891602},{"id":"https://openalex.org/C101266164","wikidata":"https://www.wikidata.org/wiki/Q2131821","display_name":"Rasch model","level":2,"score":0.7812186479568481},{"id":"https://openalex.org/C173633133","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw score","level":3,"score":0.7251653671264648},{"id":"https://openalex.org/C43122875","wikidata":"https://www.wikidata.org/wiki/Q5428522","display_name":"Facet (psychology)","level":4,"score":0.61868816614151},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5745187997817993},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5498131513595581},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.5492050051689148},{"id":"https://openalex.org/C117568660","wikidata":"https://www.wikidata.org/wiki/Q1650843","display_name":"Multinomial logistic regression","level":2,"score":0.5134197473526001},{"id":"https://openalex.org/C140390113","wikidata":"https://www.wikidata.org/wiki/Q389391","display_name":"Classical test theory","level":4,"score":0.44146019220352173},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.43675389885902405},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.42008280754089355},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.4100935161113739},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3665716052055359},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.35082024335861206},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.32284343242645264},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.3002929091453552},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.252155065536499},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.22538018226623535},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12499946355819702},{"id":"https://openalex.org/C2865642","wikidata":"https://www.wikidata.org/wiki/Q378132","display_name":"Big Five personality traits","level":3,"score":0.0},{"id":"https://openalex.org/C187288502","wikidata":"https://www.wikidata.org/wiki/Q641118","display_name":"Personality","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:libres.uncg.edu/22620","doi":null,"is_oa":false,"landing_page_url":"http://libres.uncg.edu/ir/uncg/f/Turner_uncg_0154D_12351.pdf","pdf_url":"http://libres.uncg.edu/ir/uncg/f/Turner_uncg_0154D_12351.pdf","source":{"id":"https://openalex.org/S4306401856","display_name":"NC Digital Online Collection of Knowledge and Scholarship (The University of North Carolina at Greensboro)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I169335092","host_organization_name":"University of North Carolina at Greensboro","host_organization_lineage":["https://openalex.org/I169335092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},{"id":"mag:2804034219","doi":null,"is_oa":false,"landing_page_url":"https://libres.uncg.edu/ir/uncg//listing.aspx?id=22620","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W104385083","https://openalex.org/W294218255","https://openalex.org/W623308952","https://openalex.org/W657332045","https://openalex.org/W1491613324","https://openalex.org/W1495918095","https://openalex.org/W1704942559","https://openalex.org/W1964052265","https://openalex.org/W1966576448","https://openalex.org/W1967425540","https://openalex.org/W1969548676","https://openalex.org/W1972376036","https://openalex.org/W1981835473","https://openalex.org/W1985045716","https://openalex.org/W1987153600","https://openalex.org/W1988338581","https://openalex.org/W2005261700","https://openalex.org/W2005295476","https://openalex.org/W2005950835","https://openalex.org/W2008962955","https://openalex.org/W2012950673","https://openalex.org/W2014294912","https://openalex.org/W2015934453","https://openalex.org/W2018223524","https://openalex.org/W2018606774","https://openalex.org/W2026014575","https://openalex.org/W2035370265","https://openalex.org/W2039330900","https://openalex.org/W2039456885","https://openalex.org/W2041212401","https://openalex.org/W2044059462","https://openalex.org/W2044287268","https://openalex.org/W2044848129","https://openalex.org/W2053031440","https://openalex.org/W2054100472","https://openalex.org/W2057624438","https://openalex.org/W2059813502","https://openalex.org/W2062137446","https://openalex.org/W2069942147","https://openalex.org/W2070791650","https://openalex.org/W2072352917","https://openalex.org/W2078257123","https://openalex.org/W2085271766","https://openalex.org/W2105132495","https://openalex.org/W2114608372","https://openalex.org/W2116642507","https://openalex.org/W2119479814","https://openalex.org/W2122490249","https://openalex.org/W2123555614","https://openalex.org/W2133386222","https://openalex.org/W2145770094","https://openalex.org/W2156395985","https://openalex.org/W2157623552","https://openalex.org/W2164136002","https://openalex.org/W2171234147","https://openalex.org/W2171944064","https://openalex.org/W2212838023","https://openalex.org/W2331018588","https://openalex.org/W2522205405","https://openalex.org/W2557232796","https://openalex.org/W3003342077","https://openalex.org/W4255828633"],"related_works":["https://openalex.org/W174896270","https://openalex.org/W2064572522","https://openalex.org/W1969217740","https://openalex.org/W3164476488","https://openalex.org/W7096027973","https://openalex.org/W6925275848","https://openalex.org/W4406779979","https://openalex.org/W4401877909","https://openalex.org/W2021882344","https://openalex.org/W91601597"],"abstract_inverted_index":{"In":[0,46],"the":[1,37,49,84,90,96,148,198,212,222,231,250,271,283,293,301,306,315,343,360,366,401,415,418,431,441],"peer":[2,97,118,442],"review":[3,98,443],"process":[4],"used":[5,15,61,124,167,277,319],"by":[6,174],"scientific":[7,29,100,445],"journals,":[8],"ratings":[9,91,341],"of":[10,39,51,70,86,99,104,147,197,211,221,230,241,255,264,286,308,352,379,403,417,444],"manuscripts":[11,207,287,309],"are":[12,115,374],"obtained":[13],"and":[14,54,64,78,92,107,139,154,178,192,225,243,253,260,342,409,420,436],"to":[16,28,35,62,74,82,168,248,257,278,359,399,414,421],"make":[17],"publication":[18],"decisions.":[19],"Though":[20],"concerns":[21],"have":[22,59],"been":[23,33,60],"raised":[24],"about":[25],"reviews":[26,119,378],"given":[27],"manuscripts,":[30],"little":[31],"has":[32],"done":[34],"address":[36,65],"effects":[38,85],"reviewer":[40,87,151,160,333,394,423],"severity":[41,88,152,185,334,345,367,382],"bias":[42,383],"on":[43,89],"decision":[44,171,281],"making.":[45],"other":[47,262],"settings,":[48],"methods":[50,263,426,439],"Generalizability":[52,76,136,199,407],"Theory":[53,77,137,200,408],"Many-Facet":[55,79,143,179,388,410],"Rasch":[56,80,144,180,389,411],"Measurement":[57,81,145,181,390,412],"often":[58],"investigate":[63],"such":[66],"effects.":[67],"The":[68,102,195,337],"purpose":[69],"this":[71,111],"study":[72,419],"is":[73],"use":[75],"examine":[83],"decisions":[93],"made":[94],"during":[95],"manuscripts.":[101,380,446],"merits":[103],"each":[105],"method":[106],"their":[108,377],"utility":[109],"in":[110,131,214,324,365,376,440],"novel":[112],"context":[113],"also":[114,187],"assessed.":[116],"Deidentified":[117],"(N":[120],"=":[121],"635)":[122],"that":[123,203,237,261,288,310,372],"a":[125,132,327,348],"five-item":[126],"rating":[127,438],"scale":[128],"were":[129,188,289,311,335],"included":[130],"two-facet,":[133],"partially":[134],"nested":[135,205],"analysis":[138,146,165,201],"subsequent":[140],"Decision":[141,234],"Studies.":[142],"data":[149],"produced":[150],"measures":[153,157,330,346],"manuscript":[155,170,280,302,328,363,404],"publishability":[156,215,303,316,329],"corrected":[158,182,331],"for":[159,209,219,228,332,362,429,433],"severity.":[161],"Multinomial":[162],"logistic":[163],"regression":[164],"was":[166,276,298,318,320],"compare":[169],"categories":[172],"predicted":[173],"average":[175,272,294,339],"raw":[176,191,273,295,340],"scores":[177],"scores.":[183,216,405],"Reviewer":[184,381],"rankings":[186],"compared":[189],"using":[190,292],"adjusted":[193],"methods.":[194],"results":[196],"revealed":[202],"reviewers":[204,242,373],"within":[206],"account":[208],"35.48%":[210],"variance":[213],"Manuscripts":[217],"accounted":[218,227],"12.21%":[220],"total":[223,232,274,296],"variance,":[224],"items":[226,244],"15.22%":[229],"variance.":[233],"Studies":[235],"indicated":[236],"an":[238],"unrealistic":[239],"number":[240],"would":[245],"be":[246,268,385,397],"needed":[247,398],"increase":[249],"generalizability":[251],"coefficient":[252],"index":[254],"dependability":[256],"acceptable":[258],"levels":[259],"improving":[265],"reliability":[266,402],"should":[267],"employed.":[269],"When":[270],"score":[275,297],"predict":[279],"category,":[282],"overall":[284],"percentage":[285,307],"correctly":[290,312],"classified":[291,313],"55.15%.":[299],"Using":[300],"measure":[304,317],"(theta),":[305],"when":[314],"52.49%,":[321],"suggesting":[322],"differences":[323,356],"classification,":[325],"if":[326],"used.":[336],"reviewers\u2019":[338,344],"had":[347],"Spearman":[349],"rank-order":[350],"correlation":[351],"-0.6083,":[353],"which":[354],"demonstrates":[355],"likely":[357],"attributable":[358],"adjustment":[361],"quality":[364],"measure.":[368],"These":[369,425],"findings":[370,416],"indicate":[371],"inconsistent":[375],"can":[384],"addressed":[386],"with":[387],"adjustments,":[391],"but":[392],"additional":[393],"training":[395],"may":[396],"improve":[400],"Both":[406],"contributed":[413],"understanding":[422],"behavior.":[424],"show":[427],"potential":[428],"increasing":[430],"capacity":[432],"more":[434],"fair":[435],"accurate":[437]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2018-06-01T00:00:00"}