{"id":"https://openalex.org/W2091307601","doi":"https://doi.org/10.1111/j.1743-498x.2012.00607.x","title":"A method for identifying extreme OSCE examiners","display_name":"A method for identifying extreme OSCE examiners","publication_year":2013,"publication_date":"2013-01-07","ids":{"openalex":"https://openalex.org/W2091307601","doi":"https://doi.org/10.1111/j.1743-498x.2012.00607.x","mag":"2091307601","pmid":"https://pubmed.ncbi.nlm.nih.gov/23294740"},"language":"en","primary_location":{"id":"doi:10.1111/j.1743-498x.2012.00607.x","doi":"https://doi.org/10.1111/j.1743-498x.2012.00607.x","is_oa":false,"landing_page_url":"https://doi.org/10.1111/j.1743-498x.2012.00607.x","pdf_url":null,"source":{"id":"https://openalex.org/S2280673","display_name":"The Clinical Teacher","issn_l":"1743-4971","issn":["1743-4971","1743-498X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-40","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Clinical Teacher","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037038534","display_name":"Ilona Bartman","orcid":"https://orcid.org/0000-0002-2056-479X","observed_orcids":["https://orcid.org/0000-0002-2056-479X"]},"institutions":[{"id":"https://openalex.org/I105696783","display_name":"Medical Council of Canada","ror":"https://ror.org/02vf3h353","country_code":"CA","type":"nonprofit","lineage":["https://openalex.org/I105696783"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Ilona Bartman","raw_affiliation_strings":["Evaluation Bureau, Medical Council of Canada, Ottawa, Ontario K1G 5A2, Canada. ilona@mcc.ca","Evaluation Bureau, Medical Council of Canada, Ottawa, Ontario K1G 5A2, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Evaluation Bureau, Medical Council of Canada, Ottawa, Ontario K1G 5A2, Canada. ilona@mcc.ca","institution_ids":["https://openalex.org/I105696783"]},{"raw_affiliation_string":"Evaluation Bureau, Medical Council of Canada, Ottawa, Ontario K1G 5A2, Canada","institution_ids":["https://openalex.org/I105696783"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113657027","display_name":"Sydney M. Smee","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sydney Smee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5028534070","display_name":"Marguerite Roy","orcid":"https://orcid.org/0000-0001-8462-7721","observed_orcids":["https://orcid.org/0000-0001-8462-7721"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marguerite Roy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5037038534"],"corresponding_institution_ids":["https://openalex.org/I105696783"],"apc_list":{"value":3620,"currency":"USD","value_usd":3620},"apc_paid":null,"fwci":11.2969,"has_fulltext":false,"cited_by_count":37,"citation_normalized_percentile":{"value":0.97838656,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":"10","issue":"1","first_page":"27","last_page":"31"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12602","display_name":"Medical Education and Admissions","score":0.0032999999821186066,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.0020000000949949026,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.7250000238418579},{"id":"https://openalex.org/keywords/objective-structured-clinical-examination","display_name":"OSCE","score":0.796999990940094},{"id":"https://openalex.org/keywords/cohort-effects","display_name":"cohort effects","score":0.9409999847412109},{"id":"https://openalex.org/keywords/quality-assurance","display_name":"quality assurance","score":0.5550000071525574},{"id":"https://openalex.org/keywords/high-stakes-testing","display_name":"high-stakes testing","score":0.5960000157356262}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5983238816261292},{"id":"https://openalex.org/C72563966","wikidata":"https://www.wikidata.org/wiki/Q1303415","display_name":"Cohort","level":2,"score":0.5068855881690979},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.4531165063381195},{"id":"https://openalex.org/C22679943","wikidata":"https://www.wikidata.org/wiki/Q159375","display_name":"Standard deviation","level":2,"score":0.4451771378517151},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.4383627474308014},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.4200161397457123},{"id":"https://openalex.org/C2778000598","wikidata":"https://www.wikidata.org/wiki/Q11237593","display_name":"Objective structured clinical examination","level":2,"score":0.4152681827545166},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.36005163192749023},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.34742599725723267},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.24524685740470886},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.16650328040122986},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1195126473903656},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.1089712381362915},{"id":"https://openalex.org/C2778618615","wikidata":"https://www.wikidata.org/wiki/Q4008393","display_name":"External quality assessment","level":2,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D002983","descriptor_name":"Clinical Competence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D004501","descriptor_name":"Education, Medical","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015588","descriptor_name":"Observer Variation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":2,"locations":[{"id":"doi:10.1111/j.1743-498x.2012.00607.x","doi":"https://doi.org/10.1111/j.1743-498x.2012.00607.x","is_oa":false,"landing_page_url":"https://doi.org/10.1111/j.1743-498x.2012.00607.x","pdf_url":null,"source":{"id":"https://openalex.org/S2280673","display_name":"The Clinical Teacher","issn_l":"1743-4971","issn":["1743-4971","1743-498X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-40","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Clinical Teacher","raw_type":"journal-article"},{"id":"pmid:23294740","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/23294740","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The clinical teacher","raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W590879719","https://openalex.org/W1518222008","https://openalex.org/W1526475988","https://openalex.org/W1751360006","https://openalex.org/W1993765612","https://openalex.org/W2022821166","https://openalex.org/W2031876449","https://openalex.org/W2041542884","https://openalex.org/W2076200247","https://openalex.org/W2080569127","https://openalex.org/W2114608372","https://openalex.org/W2128651242","https://openalex.org/W2139502469"],"related_works":["https://openalex.org/W2525090661","https://openalex.org/W7076196092","https://openalex.org/W1554461122","https://openalex.org/W2603453537","https://openalex.org/W4387661530","https://openalex.org/W2905692001","https://openalex.org/W6958714060","https://openalex.org/W2153692386","https://openalex.org/W7095779152","https://openalex.org/W3198986918"],"abstract_inverted_index":{"BACKGROUND:":[0],"Performance":[1],"assessments":[2],"rely":[3,75],"on":[4,76,192,202],"human":[5,77],"judgment,":[6],"and":[7,196],"are":[8,88,210,214,244],"vulnerable":[9],"to":[10,69,159,219,228,251,257],"rater":[11,30,93,102,156,235,260],"effects":[12],"(e.g.":[13],"leniency":[14],"or":[15,46,115],"harshness).":[16],"Making":[17],"valid":[18],"inferences":[19],"from":[20],"performance":[21,188],"ratings":[22],"for":[23,35,58,139],"high-stakes":[24],"decisions":[25],"requires":[26],"the":[27,54,96,118,126,136,140,149,155,184,197,217,230,234,246,253],"management":[28],"of":[29,53,72,98,128,151,199,255],"effects.":[31],"A":[32,101],"simple":[33],"method":[34,81],"detecting":[36],"extreme":[37,86,105,130,182,259],"raters":[38,87,201,213],"that":[39,74,161],"does":[40],"not":[41],"require":[42],"sophisticated":[43],"statistical":[44],"knowledge":[45],"software":[47],"has":[48,82],"been":[49,178],"developed":[50],"as":[51,180],"part":[52],"quality":[55],"assurance":[56],"process":[57],"objective":[59],"structured":[60],"clinical":[61],"examinations":[62,73],"(OSCEs).":[63],"We":[64],"believe":[65],"it":[66],"is":[67,103,123,157,165,189,236],"applicable":[68],"a":[70,145,193],"range":[71],"raters.":[78,100],"METHODS:":[79],"The":[80],"three":[83,110],"steps.":[84],"First,":[85],"identified":[89,179],"by":[90,154,216],"comparing":[91],"individual":[92],"means":[94],"with":[95,135],"mean":[97,108],"all":[99],"deemed":[104],"if":[106],"their":[107,221],"was":[109,133],"standard":[111],"deviations":[112],"below":[113],"(hawks)":[114],"above":[116],"(doves)":[117],"overall":[119,137],"mean.":[120],"This":[121,142],"criterion":[122],"adjustable.":[124],"Second,":[125],"distribution":[127,138],"an":[129],"rater's":[131,231],"scores":[132],"compared":[134],"station.":[141],"step":[143],"mitigates":[144],"station":[146],"effect.":[147],"Third,":[148],"cohort":[150,163],"candidates":[152],"seen":[153],"examined":[158],"ensure":[160],"any":[162],"effect":[164],"ruled":[166],"out.":[167],"RESULTS":[168],"AND":[169],"IMPLICATIONS:":[170],"Of":[171],"3000+":[172],"raters,":[173],"fewer":[174],"than":[175],"0.3%":[176],"have":[177],"being":[181,190],"using":[183],"proposed":[185],"criteria.":[186],"Rater":[187],"monitored":[191],"regular":[194],"basis,":[195],"impact":[198],"these":[200],"candidate":[203],"results":[204,209],"will":[205,248],"be":[206],"considered":[207],"before":[208],"finalised.":[211],"Extreme":[212],"contacted":[215],"organisation":[218,247],"review":[220],"rating":[222],"style.":[223],"If":[224],"this":[225],"intervention":[226],"fails":[227],"modify":[229],"scoring":[232],"pattern,":[233],"no":[237],"longer":[238],"invited":[239],"back.":[240],"As":[241],"more":[242],"data":[243],"collected":[245],"assess":[249],"them":[250],"inform":[252],"development":[254],"approaches":[256],"improve":[258],"performance.":[261]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":5},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2016-06-24T00:00:00"}