{"id":"https://openalex.org/W1977050797","doi":"https://doi.org/10.1097/00001888-200308000-00021","title":"Measuring Inter-rater Reliability of the Sequenced Performance Inventory and Reflective Assessment of Learning (SPIRAL)","display_name":"Measuring Inter-rater Reliability of the Sequenced Performance Inventory and Reflective Assessment of Learning (SPIRAL)","publication_year":2003,"publication_date":"2003-08-01","ids":{"openalex":"https://openalex.org/W1977050797","doi":"https://doi.org/10.1097/00001888-200308000-00021","mag":"1977050797","pmid":"https://pubmed.ncbi.nlm.nih.gov/12915381"},"language":"en","primary_location":{"id":"doi:10.1097/00001888-200308000-00021","doi":"https://doi.org/10.1097/00001888-200308000-00021","is_oa":false,"landing_page_url":"https://doi.org/10.1097/00001888-200308000-00021","pdf_url":null,"source":{"id":"https://openalex.org/S19793444","display_name":"Academic Medicine","issn_l":"1040-2446","issn":["1040-2446","1938-808X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315671","host_organization_name":"Lippincott Williams & Wilkins","host_organization_lineage":["https://openalex.org/P4310315671","https://openalex.org/P4310318547"],"host_organization_lineage_names":["Lippincott Williams & Wilkins","Wolters Kluwer"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Academic Medicine","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088543572","display_name":"Linda M. Olson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24571045","display_name":"University of North Dakota","ror":"https://ror.org/04a5szx83","country_code":"US","type":"education","lineage":["https://openalex.org/I24571045"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Linda Olson","raw_affiliation_strings":["Office of Medical Education, University of North Dakota School of Medicine and Health Sciences, Grand Forks, 58202-9037, USA. liolson@medicine.nodak.edu","e-mail: \u2329[email\u00a0protected]\u232a","requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","telephone: (701) 777-3953"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Office of Medical Education, University of North Dakota School of Medicine and Health Sciences, Grand Forks, 58202-9037, USA. liolson@medicine.nodak.edu","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"e-mail: \u2329[email\u00a0protected]\u232a","institution_ids":[]},{"raw_affiliation_string":"requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"telephone: (701) 777-3953","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004202613","display_name":"A. Dean Schieve","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24571045","display_name":"University of North Dakota","ror":"https://ror.org/04a5szx83","country_code":"US","type":"education","lineage":["https://openalex.org/I24571045"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"A. Dean Schieve","raw_affiliation_strings":["and requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037; telephone: (701) 777-3953; e-mail: \u2329[email\u00a0protected]\u232a","e-mail: \u2329[email\u00a0protected]\u232a","requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","telephone: (701) 777-3953"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"and requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037; telephone: (701) 777-3953; e-mail: \u2329[email\u00a0protected]\u232a","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"e-mail: \u2329[email\u00a0protected]\u232a","institution_ids":[]},{"raw_affiliation_string":"requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"telephone: (701) 777-3953","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108732866","display_name":"Kenneth G. Ruit","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24571045","display_name":"University of North Dakota","ror":"https://ror.org/04a5szx83","country_code":"US","type":"education","lineage":["https://openalex.org/I24571045"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenneth G. Ruit","raw_affiliation_strings":["and requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037; telephone: (701) 777-3953; e-mail: \u2329[email\u00a0protected]\u232a","e-mail: \u2329[email\u00a0protected]\u232a","requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","telephone: (701) 777-3953"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"and requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037; telephone: (701) 777-3953; e-mail: \u2329[email\u00a0protected]\u232a","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"e-mail: \u2329[email\u00a0protected]\u232a","institution_ids":[]},{"raw_affiliation_string":"requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"telephone: (701) 777-3953","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037190779","display_name":"Richard C. Vari","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I24571045","display_name":"University of North Dakota","ror":"https://ror.org/04a5szx83","country_code":"US","type":"education","lineage":["https://openalex.org/I24571045"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Richard C. Vari","raw_affiliation_strings":["and requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037; telephone: (701) 777-3953; e-mail: \u2329[email\u00a0protected]\u232a","e-mail: \u2329[email\u00a0protected]\u232a","requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","telephone: (701) 777-3953"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"and requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037; telephone: (701) 777-3953; e-mail: \u2329[email\u00a0protected]\u232a","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"e-mail: \u2329[email\u00a0protected]\u232a","institution_ids":[]},{"raw_affiliation_string":"requests for reprints should be addressed to Dr. Olson, Director, Office of Medical Education, 501 N. Columbia Road, University of North Dakota School of Medicine and Health Sciences, Grand Forks, ND 58202-9037","institution_ids":["https://openalex.org/I24571045"]},{"raw_affiliation_string":"telephone: (701) 777-3953","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5088543572"],"corresponding_institution_ids":["https://openalex.org/I24571045"],"apc_list":null,"apc_paid":null,"fwci":5.4391,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.94877278,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":95},"biblio":{"volume":"78","issue":"8","first_page":"844","last_page":"850"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.8163999915122986,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.8163999915122986,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11299","display_name":"Problem and Project Based Learning","score":0.1404000073671341,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.006599999964237213,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.9010000228881836},{"id":"https://openalex.org/keywords/spirals","display_name":"spirals","score":0.7739999890327454},{"id":"https://openalex.org/keywords/rubrics","display_name":"rubrics","score":0.8519999980926514},{"id":"https://openalex.org/keywords/problem-based-learning","display_name":"problem-based learning","score":0.906000018119812},{"id":"https://openalex.org/keywords/competency-assessment","display_name":"competency assessment","score":0.8980000019073486},{"id":"https://openalex.org/keywords/medical-students","display_name":"medical students","score":0.6299999952316284},{"id":"https://openalex.org/keywords/pbl-tutors","display_name":"PBL tutors","score":0.7200000286102295},{"id":"https://openalex.org/keywords/peer-teaching","display_name":"peer teaching","score":0.48500001430511475},{"id":"https://openalex.org/keywords/professional-behavior","display_name":"professional conduct","score":0.9319999814033508},{"id":"https://openalex.org/keywords/competency-levels","display_name":"competency levels","score":0.3499999940395355},{"id":"https://openalex.org/keywords/criterion-validity","display_name":"criterion validity","score":0.5339999794960022}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9347802996635437},{"id":"https://openalex.org/C160798450","wikidata":"https://www.wikidata.org/wiki/Q4230870","display_name":"Concordance","level":2,"score":0.7715592384338379},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6605327725410461},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5474338531494141},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.5445277690887451},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.534859299659729},{"id":"https://openalex.org/C22156102","wikidata":"https://www.wikidata.org/wiki/Q5341294","display_name":"Educational measurement","level":3,"score":0.5050228238105774},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4752858877182007},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4351261854171753},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.43014758825302124},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3162308931350708},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.280656635761261},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.18693244457244873},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.17469468712806702},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.08885449171066284},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D002983","descriptor_name":"Clinical Competence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D004504","descriptor_name":"Education, Medical, Undergraduate","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D004521","descriptor_name":"Educational Measurement","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007858","descriptor_name":"Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009658","descriptor_name":"North Dakota","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012577","descriptor_name":"Schools, Medical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015588","descriptor_name":"Observer Variation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D018794","descriptor_name":"Problem-Based Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":3,"locations":[{"id":"doi:10.1097/00001888-200308000-00021","doi":"https://doi.org/10.1097/00001888-200308000-00021","is_oa":false,"landing_page_url":"https://doi.org/10.1097/00001888-200308000-00021","pdf_url":null,"source":{"id":"https://openalex.org/S19793444","display_name":"Academic Medicine","issn_l":"1040-2446","issn":["1040-2446","1938-808X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315671","host_organization_name":"Lippincott Williams & Wilkins","host_organization_lineage":["https://openalex.org/P4310315671","https://openalex.org/P4310318547"],"host_organization_lineage_names":["Lippincott Williams & Wilkins","Wolters Kluwer"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Academic Medicine","raw_type":"journal-article"},{"id":"pmid:12915381","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/12915381","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Academic medicine : journal of the Association of American Medical Colleges","raw_type":"Evaluation Study"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.514.5143","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.514.5143","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://library.bjmu.edu.cn/pbl/document/Acad Med_2003_78(8)_844.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.9235000014305115}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W67074106","https://openalex.org/W131217486","https://openalex.org/W147476069","https://openalex.org/W1493628934","https://openalex.org/W1569154263","https://openalex.org/W1598244225","https://openalex.org/W1604560503","https://openalex.org/W1928920206","https://openalex.org/W1987388169","https://openalex.org/W1990305760","https://openalex.org/W2002664886","https://openalex.org/W2004721320","https://openalex.org/W2045078715","https://openalex.org/W2063848904","https://openalex.org/W2068700581","https://openalex.org/W2085067988","https://openalex.org/W2104626092","https://openalex.org/W2113344247","https://openalex.org/W4205604250"],"related_works":["https://openalex.org/W7119507082","https://openalex.org/W2947958355","https://openalex.org/W7220193529","https://openalex.org/W7142496565","https://openalex.org/W6945362535","https://openalex.org/W2011951181","https://openalex.org/W2161512593","https://openalex.org/W4324316242","https://openalex.org/W2925124747","https://openalex.org/W7029450819"],"abstract_inverted_index":{"PURPOSE:":[0],"To":[1],"measure":[2],"the":[3,7,50,64,85,89,134,144,164,168,171,176,180,209,225,230],"inter-rater":[4,156,184],"reliability":[5],"of":[6,14,33,52,56,72,74,84,94,116,150,170,197,208,224],"Sequenced":[8],"Performance":[9],"Inventory":[10],"and":[11,26,37,40,58,99],"Reflective":[12],"Assessment":[13],"Learning":[15],"(SPIRAL),":[16],"a":[17,44,67,97],"twenty-three":[18],"item":[19],"scoring":[20,185,202,235],"rubric":[21,236],"designed":[22],"to":[23,132,154,162,166,192,222],"assess":[24],"first":[25],"second-year":[27],"students'":[28],"competencies":[29],"such":[30],"as":[31,229],"\"acquisition":[32],"knowledge,\"":[34],"\"peer":[35],"teaching":[36],"communication":[38],"skills,\"":[39],"\"professional":[41],"behavior\"":[42],"in":[43,119,122,143],"problem-based":[45],"learning":[46],"(PBL)":[47],"curriculum":[48],"at":[49],"University":[51],"North":[53],"Dakota":[54],"School":[55],"Medicine":[57],"Health":[59],"Sciences.":[60],"METHOD:":[61],"In":[62],"2001,":[63],"authors":[65,90],"constructed":[66],"69-item":[68],"multiple-choice":[69],"questionnaire":[70],"consisting":[71],"descriptions":[73,93,107],"\"prototypical\"":[75],"(representing":[76],"real":[77,117],"students)":[78],"PBL":[79,114],"student":[80,102],"performances.":[81],"For":[82],"each":[83,207],"23":[86,210],"SPIRAL":[87,140,211,227],"items,":[88,212],"chose":[91],"competency":[92,146],"an":[95,100],"\"emerging,\"":[96],"\"developing,\"":[98],"\"advanced\"":[101],"(69":[103],"prototypical":[104,136],"students).":[105],"These":[106],"were":[108,129,159],"obtained":[109],"from":[110],"narrative":[111],"comments":[112],"by":[113,179],"facilitators":[115,126,173],"students":[118,137,220],"their":[120],"classes":[121],"2000-2001.":[123],"Seventeen":[124],"experienced":[125],"(PBL":[127],"tutors)":[128],"subsequently":[130],"asked":[131],"rank":[133],"69":[135,226],"based":[138],"on":[139],"anchor":[141],"points":[142],"three":[145],"levels.":[147],"Kendall's":[148],"coefficient":[149,196],"concordance":[151,198],"was":[152,190,203],"used":[153],"test":[155],"reliability.":[157],"Modes":[158],"also":[160,204],"determined":[161,205],"illustrate":[163],"extent":[165],"which":[167],"ratings":[169,177],"17":[172],"aligned":[174],"with":[175,213],"anticipated":[178],"authors.":[181],"RESULTS:":[182],"Overall,":[183],"for":[186,206],"all":[187],"items":[188],"combined":[189],"considered":[191],"be":[193],"reliable":[194],"(Kendall's":[195],"W":[199],"=.75).":[200],"Inter-rater":[201],"Ws":[214],"ranging":[215],"from.97":[216],"to.53.":[217],"Facilitators":[218],"rated":[219],"according":[221],"55":[223],"descriptors":[228],"researchers":[231],"anticipated.":[232],"CONCLUSIONS:":[233],"The":[234],"is":[237,243],"reliable.":[238],"Further":[239],"criterion-related":[240],"validation":[241],"study":[242],"warranted.":[244]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":1}],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2016-06-24T00:00:00"}