{"id":"https://openalex.org/W2275436624","doi":"https://doi.org/10.1080/10401334.2015.1107489","title":"Selecting and Simplifying: Rater Performance and Behavior When Considering Multiple Competencies","display_name":"Selecting and Simplifying: Rater Performance and Behavior When Considering Multiple Competencies","alternate_titles":[],"publication_year":2016,"publication_date":"2016-01-02","ids":{"openalex":"https://openalex.org/W2275436624","doi":"https://doi.org/10.1080/10401334.2015.1107489","mag":"2275436624","pmid":"https://pubmed.ncbi.nlm.nih.gov/26787084"},"language":"en","primary_location":{"id":"doi:10.1080/10401334.2015.1107489","doi":"https://doi.org/10.1080/10401334.2015.1107489","is_oa":false,"landing_page_url":"https://doi.org/10.1080/10401334.2015.1107489","pdf_url":null,"source":{"id":"https://openalex.org/S19371312","display_name":"Teaching and Learning in Medicine","issn_l":"1040-1334","issn":["1040-1334","1532-8015"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-d","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Teaching and Learning in Medicine","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W2275436624"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032335247","display_name":"Walter Tavares","orcid":"https://orcid.org/0000-0001-8267-9448","observed_orcids":["https://orcid.org/0000-0001-8267-9448"]},"institutions":[{"id":"https://openalex.org/I201744698","display_name":"Centennial College","ror":"https://ror.org/05bhp3g52","country_code":"CA","type":"education","lineage":["https://openalex.org/I201744698"]},{"id":"https://openalex.org/I98251732","display_name":"McMaster University","ror":"https://ror.org/02fa3aq29","country_code":"CA","type":"education","lineage":["https://openalex.org/I98251732"]}],"countries":["CA"],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Walter Tavares","raw_affiliation_strings":["a School of Community and Health Studies, Centennial College , Toronto , Ontario , Canada","b Division of Emergency Medicine, McMaster University , Hamilton , Ontario , Canada","Division of Emergency Medicine, McMaster University, Hamilton, Ontario, Canada","School of Community and Health Studies, Centennial College, Toronto, Ontario, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"a School of Community and Health Studies, Centennial College , Toronto , Ontario , Canada","institution_ids":["https://openalex.org/I201744698"]},{"raw_affiliation_string":"b Division of Emergency Medicine, McMaster University , Hamilton , Ontario , Canada","institution_ids":["https://openalex.org/I98251732"]},{"raw_affiliation_string":"Division of Emergency Medicine, McMaster University, Hamilton, Ontario, Canada","institution_ids":["https://openalex.org/I98251732"]},{"raw_affiliation_string":"School of Community and Health Studies, Centennial College, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I201744698"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015414971","display_name":"Shiphra Ginsburg","orcid":"https://orcid.org/0000-0002-4595-6650","observed_orcids":["https://orcid.org/0000-0002-4595-6650"]},"institutions":[{"id":"https://openalex.org/I98251732","display_name":"McMaster University","ror":"https://ror.org/02fa3aq29","country_code":"CA","type":"education","lineage":["https://openalex.org/I98251732"]}],"countries":["CA"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Shiphra Ginsburg","raw_affiliation_strings":["c Department of Medicine , McMaster University , Hamilton , Ontario , Canada","Department of Medicine, McMaster University, Hamilton, Ontario, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"c Department of Medicine , McMaster University , Hamilton , Ontario , Canada","institution_ids":["https://openalex.org/I98251732"]},{"raw_affiliation_string":"Department of Medicine, McMaster University, Hamilton, Ontario, Canada","institution_ids":["https://openalex.org/I98251732"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030421646","display_name":"Kevin W. Eva","orcid":"https://orcid.org/0000-0002-8672-2500","observed_orcids":["https://orcid.org/0000-0002-8672-2500"]},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]}],"countries":["CA"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Kevin W. Eva","raw_affiliation_strings":["d Department of Medicine , University of British Columbia, Vancouver , British Columbia , Canada","Department of Medicine, University of British Columbia, Vancouver, British Columbia, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"d Department of Medicine , University of British Columbia, Vancouver , British Columbia , Canada","institution_ids":["https://openalex.org/I141945490"]},{"raw_affiliation_string":"Department of Medicine, University of British Columbia, Vancouver, British Columbia, Canada","institution_ids":["https://openalex.org/I141945490"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5032335247"],"corresponding_institution_ids":["https://openalex.org/I201744698","https://openalex.org/I98251732"],"apc_list":null,"apc_paid":null,"fwci":19.5177,"has_fulltext":false,"cited_by_count":67,"citation_normalized_percentile":{"value":0.99294101,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":86,"max":99},"biblio":{"volume":"28","issue":"1","first_page":"41","last_page":"51"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.8906000256538391,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.8906000256538391,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12574","display_name":"Clinical Reasoning and Diagnostic Skills","score":0.06480000168085098,"subfield":{"id":"https://openalex.org/subfields/2714","display_name":"Family Practice"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11238","display_name":"Simulation-Based Education in Healthcare","score":0.006300000008195639,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rating-quality","display_name":"rating quality","score":0.550000011920929},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.9449999928474426},{"id":"https://openalex.org/keywords/clinical-competence-assessment","display_name":"clinical competence assessment","score":0.7739999890327454},{"id":"https://openalex.org/keywords/cognitive-strategies","display_name":"cognitive strategies","score":0.6039999723434448},{"id":"https://openalex.org/keywords/distraction","display_name":"distraction","score":0.33899998664855957},{"id":"https://openalex.org/keywords/simulation-based-assessment","display_name":"simulation-based assessment","score":0.597000002861023},{"id":"https://openalex.org/keywords/rater-behavior","display_name":"rater behavior","score":0.38600000739097595},{"id":"https://openalex.org/keywords/procedural-skills","display_name":"procedural skills","score":0.5460000038146973},{"id":"https://openalex.org/keywords/medical-history","display_name":"medical history","score":0.9340000152587891},{"id":"https://openalex.org/keywords/generalizability-analysis","display_name":"generalizability analysis","score":0.7329999804496765},{"id":"https://openalex.org/keywords/rater-bias","display_name":"rater bias","score":0.3199999928474426},{"id":"https://openalex.org/keywords/rater-training","display_name":"rater training","score":0.3799999952316284}],"concepts":[{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.857816219329834},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8347480297088623},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.7491095662117004},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.6122700572013855},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.5349534749984741},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5158973932266235},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.5061129927635193},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.48321211338043213},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4507111608982086},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.38740649819374084},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.38016775250434875},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.33803874254226685},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.3085632920265198},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.2877150774002075},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.10969913005828857},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D002983","descriptor_name":"Clinical Competence","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007407","descriptor_name":"Interviews as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008875","descriptor_name":"Middle Aged","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009674","descriptor_name":"Nova Scotia","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009864","descriptor_name":"Ontario","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D014743","descriptor_name":"Videotape Recording","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1080/10401334.2015.1107489","doi":"https://doi.org/10.1080/10401334.2015.1107489","is_oa":false,"landing_page_url":"https://doi.org/10.1080/10401334.2015.1107489","pdf_url":null,"source":{"id":"https://openalex.org/S19371312","display_name":"Teaching and Learning in Medicine","issn_l":"1040-1334","issn":["1040-1334","1532-8015"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-d","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Teaching and Learning in Medicine","raw_type":"journal-article"},{"id":"pmid:26787084","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/26787084","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Teaching and learning in medicine","raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W352877705","https://openalex.org/W1057512088","https://openalex.org/W1559291375","https://openalex.org/W1605824561","https://openalex.org/W1795004830","https://openalex.org/W1810330899","https://openalex.org/W1979198222","https://openalex.org/W1987548299","https://openalex.org/W1992821426","https://openalex.org/W1996986259","https://openalex.org/W2007643807","https://openalex.org/W2008770849","https://openalex.org/W2017109985","https://openalex.org/W2019398887","https://openalex.org/W2022207877","https://openalex.org/W2036552474","https://openalex.org/W2044059462","https://openalex.org/W2059582218","https://openalex.org/W2067404301","https://openalex.org/W2068757607","https://openalex.org/W2070413143","https://openalex.org/W2073742558","https://openalex.org/W2083559293","https://openalex.org/W2084778079","https://openalex.org/W2095723554","https://openalex.org/W2101416823","https://openalex.org/W2113256441","https://openalex.org/W2130736456","https://openalex.org/W2132892830","https://openalex.org/W2145187709","https://openalex.org/W2161254436","https://openalex.org/W2164558494","https://openalex.org/W2186524655","https://openalex.org/W2573498121","https://openalex.org/W2608014122","https://openalex.org/W3146566578","https://openalex.org/W4213346165","https://openalex.org/W4241246521"],"related_works":["https://openalex.org/W2186524655","https://openalex.org/W821076729","https://openalex.org/W2321091137","https://openalex.org/W3198986918","https://openalex.org/W4387536758","https://openalex.org/W2408873607","https://openalex.org/W2964072339","https://openalex.org/W2055563246","https://openalex.org/W2074705952","https://openalex.org/W2978021061"],"abstract_inverted_index":{"THEORY:":[0],"Assessment":[1],"of":[2,43,46,81,92,112,117,123,128,137,162,173,181,239,267,283,338,361],"clinical":[3,106],"competence":[4],"is":[5,242],"a":[6,109],"complex":[7],"cognitive":[8,65,339,343,348,387],"task":[9,251],"with":[10,204],"many":[11,336,374],"mental":[12],"demands":[13,358],"often":[14],"imposed":[15],"on":[16,41,400],"raters":[17,48,67,94,241,334,368,396],"unintentionally.":[18],"We":[19,77,168],"were":[20,131,142,273],"interested":[21],"in":[22,31,165],"whether":[23],"this":[24,35],"burden":[25],"might":[26],"contribute":[27],"to":[28,49,57,71,97,133,144,158,198,209,226,246,262,275,285,292,346,372,381,385,394],"well-described":[29],"limitations":[30],"assessment":[32],"judgments.":[33],"In":[34,60],"study":[36],"we":[37,62,191],"examine":[38],"the":[39,64,90,129,153,160,166,170,174,179,200,218,227,237,258,286,351,392],"effect":[40],"indicators":[42,360],"rating":[44,82,205,250,352,357,362],"quality":[45,83,363],"asking":[47],"(a)":[50],"consider":[51,72,98],"multiple":[52,58,73],"competencies":[53,74],"and":[54,140,177,233,341,379],"(b)":[55],"attend":[56],"issues.":[59],"addition,":[61],"explored":[63],"strategies":[66,202,344,366],"engage":[68,369],"when":[69,236,370],"asked":[70,132,143,274,291,371],"simultaneously.":[75],"HYPOTHESES:":[76],"hypothesized":[78],"that":[79,217,270,367],"indications":[80],"(e.g.,":[84],"interrater":[85,171,223],"reliability)":[86],"would":[87],"decline":[88],"as":[89,186],"number":[91,180],"dimensions":[93,136,260,279,288,375],"are":[95,377],"expected":[96],"increases.":[99],"METHOD:":[100],"Experienced":[101],"faculty":[102],"examiners":[103],"rated":[104],"prerecorded":[105],"performances":[107],"within":[108],"2":[110,115,147,259,278],"(number":[111,122],"dimensions)":[113],"\u00d7":[114,120],"(presence":[116],"distracting":[118],"task)":[119],"3":[121],"videos)":[124],"factorial":[125],"design.":[126],"Half":[127],"participants":[130,184,271],"rate":[134,145,159,276,293,373],"7":[135,294],"performance":[138,161],"(7D),":[139],"half":[141],"only":[146,277],"(2D).":[148],"The":[149,244,365],"second":[150],"factor":[151],"involved":[152],"requirement":[154,245],"(or":[155],"lack":[156],"thereof)":[157],"actors":[163],"participating":[164],"simulation.":[167],"calculated":[169],"reliability":[172,224],"scores":[175],"assigned":[176],"counted":[178],"relevant":[182],"behaviors":[183,282],"identified":[185,280,335],"informing":[187],"their":[188],"ratings.":[189],"Second,":[190],"analyzed":[192],"data":[193],"from":[194],"semistructured":[195],"posttask":[196,332],"interviews":[197],"explore":[199],"rater":[201],"associated":[203],"under":[206],"conditions":[207],"designed":[208],"broaden":[210],"raters'":[211],"focus.":[212],"RESULTS:":[213],"Generalizability":[214],"analyses":[215],"revealed":[216,269],"2D":[219],"group":[220,229],"achieved":[221],"higher":[222],"relative":[225],"7D":[228],"(G":[230],"=":[231,298,315],".56":[232],".42,":[234],"respectively,":[235],"average":[238],"10":[240],"calculated).":[243],"complete":[247],"an":[248,255,265],"additional":[249],"did":[252],"not":[253],"have":[254],"effect.":[256],"Using":[257],"common":[261],"both":[263],"groups,":[264],"analysis":[266],"variance":[268],"who":[272],"more":[281],"relevance":[284],"focal":[287],"than":[289],"those":[290],"dimensions:":[295],"procedural":[296],"skill":[297],"36.2%,":[299],"95%":[300,308,317,323],"confidence":[301],"interval":[302],"(CI)":[303],"[32.5,":[304],"40.0]":[305],"versus":[306,321],"23.5%,":[307],"CI":[309,318,324],"[20.8,":[310],"26.3],":[311],"respectively;":[312,327],"history":[313],"gathering":[314],"38.6%,":[316],"[33.5,":[319],"42.9]":[320],"24.0%,":[322],"[21.1,":[325],"26.9],":[326],"ps":[328],"<":[329],".05.":[330],"During":[331],"interviews,":[333],"sources":[337],"load":[340,349],"idiosyncratic":[342,383],"used":[345],"reduce":[347,386],"during":[350],"task.":[353],"CONCLUSIONS:":[354],"As":[355],"intrinsic":[356],"increase,":[359],"decline.":[364],"simultaneously":[376],"varied":[378],"appear":[380],"yield":[382],"efforts":[384],"effort,":[388],"which":[389,395],"may":[390],"affect":[391],"degree":[393],"make":[397],"judgments":[398],"based":[399],"comparable":[401],"information.":[402]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2275436624","counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":14},{"year":2019,"cited_by_count":11},{"year":2018,"cited_by_count":11},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":3},{"year":2014,"cited_by_count":1}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2016-06-24T00:00:00"}
