{"id":"https://openalex.org/W2931259641","doi":"https://doi.org/10.1177/0013164419834613","title":"Exploring the Combined Effects of Rater Misfit and Differential Rater Functioning in Performance Assessments","display_name":"Exploring the Combined Effects of Rater Misfit and Differential Rater Functioning in Performance Assessments","publication_year":2019,"publication_date":"2019-04-02","ids":{"openalex":"https://openalex.org/W2931259641","doi":"https://doi.org/10.1177/0013164419834613","mag":"2931259641","pmid":"https://pubmed.ncbi.nlm.nih.gov/31488921","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/6713980"},"language":"en","primary_location":{"id":"doi:10.1177/0013164419834613","doi":"https://doi.org/10.1177/0013164419834613","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0013164419834613","pdf_url":null,"source":{"id":"https://openalex.org/S176831015","display_name":"Educational and Psychological Measurement","issn_l":"0013-1644","issn":["0013-1644","1552-3888"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","dongbi-b","doyens","erih-plus","jufo-2","ki-jl-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational and Psychological Measurement","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/6713980","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026354052","display_name":"Stefanie A. Wind","orcid":"https://orcid.org/0000-0002-1599-375X","observed_orcids":["https://orcid.org/0000-0002-1599-375X"]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Stefanie A. Wind","raw_affiliation_strings":["The University of Alabama, Tuscaloosa, AL, USA"],"raw_orcid":"https://orcid.org/0000-0002-1599-375X","affiliations":[{"raw_affiliation_string":"The University of Alabama, Tuscaloosa, AL, USA","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080748347","display_name":"Wenjing Guo","orcid":"https://orcid.org/0000-0001-8271-9374","observed_orcids":["https://orcid.org/0000-0001-8271-9374"]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wenjing Guo","raw_affiliation_strings":["The University of Alabama, Tuscaloosa, AL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Alabama, Tuscaloosa, AL, USA","institution_ids":["https://openalex.org/I17301866"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5026354052"],"corresponding_institution_ids":["https://openalex.org/I17301866"],"apc_list":{"value":3650,"currency":"USD","value_usd":3650},"apc_paid":null,"fwci":2.895,"has_fulltext":false,"cited_by_count":29,"citation_normalized_percentile":{"value":0.90582937,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":98},"biblio":{"volume":"79","issue":"5","first_page":"962","last_page":"987"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.9390000104904175,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.9390000104904175,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.011599999852478504,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.006800000090152025,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.8640000224113464},{"id":"https://openalex.org/keywords/performance-evaluation-human-resources","display_name":"performance evaluation (human resources)","score":0.8029999732971191},{"id":"https://openalex.org/keywords/rater-bias","display_name":"rater bias","score":0.8040000200271606},{"id":"https://openalex.org/keywords/rater-calibration","display_name":"rater calibration","score":0.38499999046325684},{"id":"https://openalex.org/keywords/student-subgroups","display_name":"student subgroups","score":0.5659999847412109}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.7193191051483154},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6455636620521545},{"id":"https://openalex.org/C3019921246","wikidata":"https://www.wikidata.org/wiki/Q104853310","display_name":"Differential effects","level":2,"score":0.48765456676483154},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.4616839289665222},{"id":"https://openalex.org/C93226319","wikidata":"https://www.wikidata.org/wiki/Q193137","display_name":"Differential (mechanical device)","level":2,"score":0.4133767783641815},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.3615230321884155},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.33845841884613037},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.23888900876045227},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.22668465971946716},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.10625272989273071},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0897161066532135},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1177/0013164419834613","doi":"https://doi.org/10.1177/0013164419834613","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0013164419834613","pdf_url":null,"source":{"id":"https://openalex.org/S176831015","display_name":"Educational and Psychological Measurement","issn_l":"0013-1644","issn":["0013-1644","1552-3888"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","dongbi-b","doyens","erih-plus","jufo-2","ki-jl-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational and Psychological Measurement","raw_type":"journal-article"},{"id":"pmh:oai:pubmedcentral.nih.gov:6713980","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/6713980","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educ Psychol Meas","raw_type":"Text"},{"id":"pmid:31488921","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/31488921","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational and psychological measurement","raw_type":"Journal Article"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:6713980","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/6713980","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educ Psychol Meas","raw_type":"Text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W121592247","https://openalex.org/W615644768","https://openalex.org/W1546462660","https://openalex.org/W1955607021","https://openalex.org/W1968999604","https://openalex.org/W1972883603","https://openalex.org/W1983471309","https://openalex.org/W1984970102","https://openalex.org/W2031703645","https://openalex.org/W2051039162","https://openalex.org/W2077649418","https://openalex.org/W2094426344","https://openalex.org/W2100044678","https://openalex.org/W2121720784","https://openalex.org/W2127374659","https://openalex.org/W2140051752","https://openalex.org/W2142266390","https://openalex.org/W2145770094","https://openalex.org/W2342276252","https://openalex.org/W2409918758","https://openalex.org/W2415280278","https://openalex.org/W2428946175","https://openalex.org/W2439748702","https://openalex.org/W2484407085","https://openalex.org/W2503552454","https://openalex.org/W2505213109","https://openalex.org/W2574111388","https://openalex.org/W2582743722","https://openalex.org/W2597677657","https://openalex.org/W2734337736","https://openalex.org/W2741387466","https://openalex.org/W2791758691","https://openalex.org/W2807535818","https://openalex.org/W2886473660","https://openalex.org/W2888779986","https://openalex.org/W2948210953","https://openalex.org/W3025249209","https://openalex.org/W4238804190"],"related_works":["https://openalex.org/W3035452048","https://openalex.org/W2921914495","https://openalex.org/W3125290220","https://openalex.org/W4391967592","https://openalex.org/W2886473660","https://openalex.org/W7095779152","https://openalex.org/W3198114581","https://openalex.org/W4285409866","https://openalex.org/W2948994592","https://openalex.org/W75369069"],"abstract_inverted_index":{"Rater":[0],"effects,":[1,37,164],"or":[2],"raters'":[3],"tendencies":[4],"to":[5,8,58,88,92,99,112,116,131,147,172,188,205],"assign":[6],"ratings":[7,15],"performances":[9,18],"that":[10,16,41,143,166,181],"are":[11,20,97,186,194],"different":[12,122],"from":[13],"the":[14,17,78,90,114],"warranted,":[19],"well":[21],"documented":[22],"in":[23,80],"rater-mediated":[24],"assessments":[25],"across":[26],"a":[27,50,128],"variety":[28],"of":[29,35,52,72,84,109,134,154,183,199],"disciplines.":[30],"In":[31],"many":[32],"real-data":[33],"studies":[34,71],"rater":[36,63,73,94,100,137,157,184],"researchers":[38,67],"have":[39],"reported":[40],"raters":[42,103,160],"exhibit":[43,104,161],"more":[44,105],"than":[45,106],"one":[46,107],"effect,":[47,110],"such":[48],"as":[49,202],"combination":[51],"misfit":[53,158],"and":[54,111,136,151,156,207],"systematic":[55],"biases":[56],"related":[57],"student":[59],"subgroups":[60],"(i.e.,":[61],"differential":[62],"functioning":[64],"[DRF]).":[65],"However,":[66],"who":[68],"conduct":[69],"simulation":[70,129],"effects":[74,79,101,168,185],"usually":[75],"focus":[76],"on":[77],"isolation.":[81],"The":[82],"purpose":[83],"this":[85,118],"study":[86,130],"was":[87],"explore":[89,113,132],"degree":[91,115],"which":[93,117],"effect":[95],"indicators":[96,153],"sensitive":[98],"when":[102,159,190],"type":[108],"sensitivity":[119],"changes":[120],"under":[121],"data":[123],"collection":[124],"designs.":[125],"We":[126,178,196],"used":[127],"combinations":[133,182],"DRF":[135,155],"misfit.":[138],"Overall,":[139],"our":[140,200],"findings":[141,201],"suggested":[142],"it":[144],"is":[145],"possible":[146],"use":[148],"common":[149],"numeric":[150,176],"graphical":[152],"both":[162],"these":[163,167],"but":[165],"may":[169],"be":[170],"difficult":[171],"distinguish":[173],"using":[174],"only":[175],"indicators.":[177],"also":[179],"observed":[180],"easier":[187],"identify":[189],"complete":[191],"rating":[192],"designs":[193],"used.":[195],"discuss":[197],"implications":[198],"they":[203],"result":[204],"research":[206],"practice.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2019-04-11T00:00:00"}