{"id":"https://openalex.org/W2117032496","doi":"https://doi.org/10.1177/0013164499591002","title":"Evaluating the Validity of Educational Rating Data","display_name":"Evaluating the Validity of Educational Rating Data","publication_year":1999,"publication_date":"1999-02-01","ids":{"openalex":"https://openalex.org/W2117032496","doi":"https://doi.org/10.1177/0013164499591002","mag":"2117032496"},"language":"en","primary_location":{"id":"doi:10.1177/0013164499591002","doi":"https://doi.org/10.1177/0013164499591002","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0013164499591002","pdf_url":null,"source":{"id":"https://openalex.org/S176831015","display_name":"Educational and Psychological Measurement","issn_l":"0013-1644","issn":["0013-1644","1552-3888"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","dongbi-b","doyens","erih-plus","jufo-2","ki-jl-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational and Psychological Measurement","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082759326","display_name":"Michael R. Harwell","orcid":"https://orcid.org/0000-0003-1729-9910","observed_orcids":["https://orcid.org/0000-0003-1729-9910"]},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Michael Harwell","raw_affiliation_strings":["University of Pittsburgh"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pittsburgh","institution_ids":["https://openalex.org/I170201317"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5082759326"],"corresponding_institution_ids":["https://openalex.org/I170201317"],"apc_list":{"value":3650,"currency":"USD","value_usd":3650},"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.22746781,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":93},"biblio":{"volume":"59","issue":"1","first_page":"25","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.8715999722480774,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.8715999722480774,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.11140000075101852,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10561","display_name":"Educational Research and Methods","score":0.0034000000450760126,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rater-training","display_name":"rater training","score":0.6859999895095825},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.925000011920929},{"id":"https://openalex.org/keywords/validity-evidence","display_name":"validity evidence","score":0.6190000176429749},{"id":"https://openalex.org/keywords/internal-validity","display_name":"internal validity","score":0.8930000066757202},{"id":"https://openalex.org/keywords/experimental-design","display_name":"experimental design","score":0.6460000276565552}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.7440981268882751},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.7039063572883606},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.6143770217895508},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5665221214294434},{"id":"https://openalex.org/C174106493","wikidata":"https://www.wikidata.org/wiki/Q1057880","display_name":"External validity","level":2,"score":0.5153657793998718},{"id":"https://openalex.org/C116211729","wikidata":"https://www.wikidata.org/wiki/Q5372350","display_name":"Test validity","level":3,"score":0.5044199228286743},{"id":"https://openalex.org/C70364389","wikidata":"https://www.wikidata.org/wiki/Q18757","display_name":"Validity","level":3,"score":0.4992845058441162},{"id":"https://openalex.org/C182050348","wikidata":"https://www.wikidata.org/wiki/Q5186576","display_name":"Criterion validity","level":4,"score":0.45934611558914185},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.44430646300315857},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3974166810512543},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.3818334639072418},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.3777446448802948},{"id":"https://openalex.org/C49453240","wikidata":"https://www.wikidata.org/wiki/Q1592163","display_name":"Construct validity","level":3,"score":0.2876087427139282},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.2844279110431671},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.23166126012802124},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.14201796054840088},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.12378561496734619},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1177/0013164499591002","doi":"https://doi.org/10.1177/0013164499591002","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0013164499591002","pdf_url":null,"source":{"id":"https://openalex.org/S176831015","display_name":"Educational and Psychological Measurement","issn_l":"0013-1644","issn":["0013-1644","1552-3888"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","dongbi-b","doyens","erih-plus","jufo-2","ki-jl-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational and Psychological Measurement","raw_type":"journal-article"},{"id":"doi:10.1177/00131649921969712","doi":"https://doi.org/10.1177/00131649921969712","is_oa":false,"landing_page_url":"https://doi.org/10.1177/00131649921969712","pdf_url":null,"source":{"id":"https://openalex.org/S176831015","display_name":"Educational and Psychological Measurement","issn_l":"0013-1644","issn":["0013-1644","1552-3888"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","dongbi-b","doyens","erih-plus","jufo-2","ki-jl-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational and Psychological Measurement","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8278999924659729,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W298593807","https://openalex.org/W1600912621","https://openalex.org/W1958015814","https://openalex.org/W1974613584","https://openalex.org/W1975971445","https://openalex.org/W1989688510","https://openalex.org/W1993487189","https://openalex.org/W1999912466","https://openalex.org/W2003718413","https://openalex.org/W2012710600","https://openalex.org/W2017529798","https://openalex.org/W2029098105","https://openalex.org/W2043633832","https://openalex.org/W2046846460","https://openalex.org/W2056247142","https://openalex.org/W2064396658","https://openalex.org/W2073126333","https://openalex.org/W2094264376","https://openalex.org/W2101201253","https://openalex.org/W2102847425","https://openalex.org/W2114608372","https://openalex.org/W2122229640","https://openalex.org/W2123555614","https://openalex.org/W2125705186","https://openalex.org/W2129709750","https://openalex.org/W2139678826","https://openalex.org/W2159854395","https://openalex.org/W2166831636","https://openalex.org/W2171973401","https://openalex.org/W2288572692","https://openalex.org/W2727383325","https://openalex.org/W2905010601","https://openalex.org/W4213276201"],"related_works":["https://openalex.org/W2741121305","https://openalex.org/W2895893693","https://openalex.org/W2728402506","https://openalex.org/W1515339196","https://openalex.org/W2328772763","https://openalex.org/W2143226145","https://openalex.org/W2024160177","https://openalex.org/W97833999","https://openalex.org/W3023681263","https://openalex.org/W2626348402"],"abstract_inverted_index":{"The":[0],"use":[1,21],"of":[2,15,22,24,36,48,101,112,123],"trained":[3],"raters":[4,18,27,71,91,129],"has":[5,39],"a":[6],"long":[7],"tradition":[8],"in":[9],"educational":[10],"research.":[11],"A":[12],"standard":[13],"feature":[14],"studies":[16,89,127],"employing":[17,90,128],"is":[19],"the":[20,34,45,98,102,113,121,124],"indicators":[23],"agreement":[25],"among":[26],"such":[28],"as":[29,72,109],"interrater":[30],"reliability":[31,52],"coefficients.":[32],"Surprisingly,":[33],"validity":[35,64,85,100],"rating":[37,67],"data":[38,73],"received":[40],"relatively":[41],"little":[42,54],"attention,":[43],"raising":[44],"undesirable":[46],"prospect":[47],"ratings":[49,117],"with":[50],"satisfactory":[51],"but":[53],"validity.":[55],"This":[56],"article":[57],"suggests":[58],"two":[59],"complementary":[60],"frameworks":[61],"for":[62,66,83],"providing":[63],"evidence":[65],"data.":[68],"One":[69],"conceptualizes":[70],"collection":[74],"instruments":[75],"that":[76],"should":[77],"be":[78,105],"subject":[79],"to":[80,104,115,120,132],"traditional":[81],"procedures":[82],"establishing":[84],"evidence;":[86],"another":[87],"evaluates":[88],"from":[92],"an":[93,110],"experimental":[94],"design":[95],"perspective,":[96],"permitting":[97],"internal":[99],"study":[103],"assessed":[106],"and":[107],"used":[108,131],"indicator":[111],"extent":[114],"which":[116],"are":[118,130],"attributable":[119],"training":[122],"raters.":[125],"Two":[126],"illustrate":[133],"these":[134],"ideas.":[135]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2016-06-24T00:00:00"}