{"id":"https://openalex.org/W3198986918","doi":null,"title":"Rater Stability in a High-Stakes Performance Assessment: A Longitudinal Investigation","display_name":"Rater Stability in a High-Stakes Performance Assessment: A Longitudinal Investigation","alternate_titles":[],"publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W3198986918","mag":"3198986918"},"language":"en","primary_location":{"id":"pmh:oai:figshare.com:article/10783709","doi":null,"is_oa":false,"landing_page_url":"https://figshare.com/articles/thesis/Rater_Stability_in_a_High-Stakes_Performance_Assessment_A_Longitudinal_Investigation/10783709","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Thesis"},"parent_works":[],"lineage":["https://openalex.org/W3198986918"],"child_works_count":0,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024829867","display_name":"Tara M McNaughton","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Tara M McNaughton","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5024829867"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.8574000000953674,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.8574000000953674,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.06809999793767929,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.013299999758601189,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rater-severity","display_name":"rater severity","score":0.9660000205039978},{"id":"https://openalex.org/keywords/rater-consistency","display_name":"rater consistency","score":0.9599999785423279},{"id":"https://openalex.org/keywords/performance-evaluation-human-resources","display_name":"performance evaluation (human resources)","score":0.6539999842643738},{"id":"https://openalex.org/keywords/medical-certification","display_name":"medical certification","score":0.6129999756813049},{"id":"https://openalex.org/keywords/high-stakes-testing","display_name":"high-stakes testing","score":0.6660000085830688},{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.6759999990463257},{"id":"https://openalex.org/keywords/rater-training","display_name":"rater training","score":0.6679999828338623},{"id":"https://openalex.org/keywords/many-facet-rasch-measurement","display_name":"many-facet Rasch model","score":0.6830000281333923},{"id":"https://openalex.org/keywords/partial-credit-model","display_name":"partial credit model","score":0.7360000014305115}],"concepts":[{"id":"https://openalex.org/C101266164","wikidata":"https://www.wikidata.org/wiki/Q2131821","display_name":"Rasch model","level":2,"score":0.7987154722213745},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.7041984796524048},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.5593469142913818},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5571679472923279},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.5186373591423035},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.4970090687274933},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.493965208530426},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.4390344023704529},{"id":"https://openalex.org/C106347477","wikidata":"https://www.wikidata.org/wiki/Q5384228","display_name":"Equating","level":3,"score":0.4367119073867798},{"id":"https://openalex.org/C207968926","wikidata":"https://www.wikidata.org/wiki/Q7227107","display_name":"Polytomous Rasch model","level":4,"score":0.42918336391448975},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.41458195447921753},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.40874040126800537},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.3494763672351837},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3261488080024719},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.20045113563537598},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.16151204705238342},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.11261710524559021},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:figshare.com:article/10783709","doi":null,"is_oa":false,"landing_page_url":"https://figshare.com/articles/thesis/Rater_Stability_in_a_High-Stakes_Performance_Assessment_A_Longitudinal_Investigation/10783709","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Thesis"},{"id":"mag:3198986918","doi":null,"is_oa":false,"landing_page_url":"https://oatd.org/oatd/record?record=handle\\:10027\\%2F22632","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W228400431","https://openalex.org/W2493796278","https://openalex.org/W114814995","https://openalex.org/W4394684555","https://openalex.org/W196858522","https://openalex.org/W2128811180","https://openalex.org/W1987153600","https://openalex.org/W2587690098","https://openalex.org/W7100821139","https://openalex.org/W2043544836"],"abstract_inverted_index":{"The":[0,38,124],"certification":[1],"of":[2,19,41,61,133,156,213],"medical":[3],"practitioners":[4],"frequently":[5],"includes":[6],"a":[7,86,101,129,169,217],"performance":[8,21,51,62],"assessment":[9],"to":[10,23,49,52,77,99,105,119,143,226],"ensure":[11],"competence.":[12],"Although":[13],"such":[14],"assessments":[15],"offer":[16],"richer":[17],"evaluations":[18],"examinee":[20,50],"compared":[22],"other":[24,116],"exam":[25],"types,":[26],"the":[27,42,72,91,154,211],"reliance":[28],"on":[29,90],"expert":[30],"judgement":[31],"in":[32,182],"evaluating":[33],"examinees":[34],"presents":[35],"some":[36],"concerns.":[37],"subjective":[39],"nature":[40],"rating":[43,82,228],"task":[44],"may":[45,57,223],"allow":[46],"factors":[47],"unrelated":[48],"influence":[53],"ratings,":[54],"and":[55,67,81,109,145,159],"raters":[56,134,144,164,190,209],"have":[58],"idiosyncratic":[59],"perceptions":[60],"levels.":[63],"To":[64],"assess":[65],"inter-":[66],"intra-rater":[68],"differences,":[69],"I":[70,94,152],"used":[71,95,118],"Many-Facet":[73],"Rasch":[74],"Measurement":[75],"model":[76,89],"quantify":[78],"rater":[79,92,96,122,146,157,180,197,221],"severity":[80,158,181],"scale":[83],"category":[84,97,102,125],"use.\\n\\nApplying":[85],"partial":[87],"credit":[88],"facet,":[93],"thresholds":[98],"calculate":[100],"breadth":[103,126],"measure":[104],"identify":[106,120],"central":[107],"tendency":[108],"extremism.":[110],"This":[111],"method":[112,127],"compares":[113],"favorably":[114],"with":[115,202],"indices":[117],"these":[121],"effects.":[123],"identifies":[128],"slightly":[130],"larger":[131],"proportion":[132],"as":[135,216],"exhibiting":[136],"effects":[137],"while":[138],"providing":[139],"more":[140],"precise":[141],"feedback":[142],"trainers.":[147],"Using":[148],"hierarchical":[149],"linear":[150],"models,":[151],"assessed":[153],"stability":[155],"consistency":[160,193,198],"measures":[161],"longitudinally.":[162],"Most":[163],"demonstrated":[165,191],"stable":[166,192],"severity;":[167],"however,":[168],"sizable":[170],"minority":[171],"did":[172],"not.":[173],"Therefore,":[174],"caution":[175],"is":[176],"warranted":[177],"when":[178],"using":[179],"common":[183],"element":[184],"equating":[185],"designs.":[186],"Conversely,":[187],"nearly":[188],"all":[189],"measures,":[194],"suggesting":[195],"that":[196],"does":[199],"not":[200],"improve":[201,227],"experience.":[203],"More":[204],"intensive":[205],"training":[206],"for":[207,220],"new":[208],"or":[210],"use":[212],"practice":[214],"ratings":[215],"screening":[218],"tool":[219],"selection":[222],"be":[224],"necessary":[225],"quality.":[229]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3198986918","counts_by_year":[],"updated_date":"2026-11-14T11:04:58.165337","created_date":"2021-09-27T00:00:00"}
