{"id":"https://openalex.org/W2767465286","doi":"https://doi.org/10.1080/15305058.2017.1361426","title":"The Influence of Rater Effects in Training Sets on the Psychometric Quality of Automated Scoring for Writing Assessments","display_name":"The Influence of Rater Effects in Training Sets on the Psychometric Quality of Automated Scoring for Writing Assessments","publication_year":2017,"publication_date":"2017-11-07","ids":{"openalex":"https://openalex.org/W2767465286","doi":"https://doi.org/10.1080/15305058.2017.1361426","mag":"2767465286"},"language":"en","primary_location":{"id":"doi:10.1080/15305058.2017.1361426","doi":"https://doi.org/10.1080/15305058.2017.1361426","is_oa":false,"landing_page_url":"https://doi.org/10.1080/15305058.2017.1361426","pdf_url":null,"source":{"id":"https://openalex.org/S145459003","display_name":"International Journal of Testing","issn_l":"1530-5058","issn":["1530-5058","1532-7574"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","jufo-1","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Testing","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026354052","display_name":"Stefanie A. Wind","orcid":"https://orcid.org/0000-0002-1599-375X","observed_orcids":["https://orcid.org/0000-0002-1599-375X"]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Stefanie A. Wind","raw_affiliation_strings":["The University of Alabama"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Alabama","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025303389","display_name":"Edward W. Wolfe","orcid":"https://orcid.org/0000-0002-8551-4646","observed_orcids":["https://orcid.org/0000-0002-8551-4646"]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Edward W. Wolfe","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084356454","display_name":"George Engelhard","orcid":"https://orcid.org/0000-0002-1694-8942","observed_orcids":["https://orcid.org/0000-0002-1694-8942"]},"institutions":[{"id":"https://openalex.org/I165733156","display_name":"University of Georgia","ror":"https://ror.org/00te3t702","country_code":"US","type":"education","lineage":["https://openalex.org/I165733156"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"George Engelhard","raw_affiliation_strings":["The University of Georgia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Georgia","institution_ids":["https://openalex.org/I165733156"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082669876","display_name":"Peter W. Foltz","orcid":"https://orcid.org/0000-0002-0281-8741","observed_orcids":["https://orcid.org/0000-0002-0281-8741"]},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]},{"id":"https://openalex.org/I2802236040","display_name":"University of Colorado System","ror":"https://ror.org/00jc20583","country_code":"US","type":"education","lineage":["https://openalex.org/I2802236040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Foltz","raw_affiliation_strings":["Pearson and University of Colorado, Boulder"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pearson and University of Colorado, Boulder","institution_ids":["https://openalex.org/I188538660","https://openalex.org/I2802236040"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020252972","display_name":"Mark Rosenstein","orcid":"https://orcid.org/0000-0002-1258-5064","observed_orcids":["https://orcid.org/0000-0002-1258-5064"]},"institutions":[{"id":"https://openalex.org/I1283104182","display_name":"Pearson (United States)","ror":"https://ror.org/02naadr48","country_code":"US","type":"company","lineage":["https://openalex.org/I1283104182","https://openalex.org/I4210088969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Rosenstein","raw_affiliation_strings":["Pearson"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pearson","institution_ids":["https://openalex.org/I1283104182"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5026354052"],"corresponding_institution_ids":["https://openalex.org/I17301866"],"apc_list":null,"apc_paid":null,"fwci":1.4541,"has_fulltext":false,"cited_by_count":33,"citation_normalized_percentile":{"value":0.84374147,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"18","issue":"1","first_page":"27","last_page":"49"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.46630001068115234,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.46630001068115234,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.2809999883174896,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.04529999941587448,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-essay-scoring","display_name":"automated essay scoring","score":0.9020000100135803},{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.9909999966621399},{"id":"https://openalex.org/keywords/training-data-quality","display_name":"training data quality","score":0.6949999928474426},{"id":"https://openalex.org/keywords/human-ratings","display_name":"human ratings","score":0.718999981880188},{"id":"https://openalex.org/keywords/writing-assessment","display_name":"writing assessment","score":0.7919999957084656},{"id":"https://openalex.org/keywords/psychometric-properties","display_name":"psychometric properties","score":0.6919999718666077},{"id":"https://openalex.org/keywords/rater-mediated-assessment","display_name":"rater-mediated assessment","score":0.546999990940094},{"id":"https://openalex.org/keywords/language-assessment","display_name":"language assessment","score":0.5139999985694885}],"concepts":[{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.77997225522995},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5918471813201904},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.5587874054908752},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5396710634231567},{"id":"https://openalex.org/C2777898490","wikidata":"https://www.wikidata.org/wiki/Q17157236","display_name":"Writing assessment","level":2,"score":0.5371179580688477},{"id":"https://openalex.org/C2779346075","wikidata":"https://www.wikidata.org/wiki/Q7268763","display_name":"Quality Score","level":3,"score":0.5045195817947388},{"id":"https://openalex.org/C53811970","wikidata":"https://www.wikidata.org/wiki/Q5062194","display_name":"Centrality","level":2,"score":0.4582812190055847},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.4545407295227051},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.4459885060787201},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4143708646297455},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4123048484325409},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.15013867616653442},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1435593068599701},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.12607553601264954},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1080/15305058.2017.1361426","doi":"https://doi.org/10.1080/15305058.2017.1361426","is_oa":false,"landing_page_url":"https://doi.org/10.1080/15305058.2017.1361426","pdf_url":null,"source":{"id":"https://openalex.org/S145459003","display_name":"International Journal of Testing","issn_l":"1530-5058","issn":["1530-5058","1532-7574"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","jufo-1","norway-1","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Testing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.4156000018119812}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W36899101","https://openalex.org/W265329213","https://openalex.org/W562804573","https://openalex.org/W643903114","https://openalex.org/W657332045","https://openalex.org/W1511133876","https://openalex.org/W1554944419","https://openalex.org/W1593930920","https://openalex.org/W1920251231","https://openalex.org/W1954968711","https://openalex.org/W1966576448","https://openalex.org/W1975971445","https://openalex.org/W2024036411","https://openalex.org/W2044762408","https://openalex.org/W2076540717","https://openalex.org/W2088696944","https://openalex.org/W2100044678","https://openalex.org/W2145770094","https://openalex.org/W2414642490","https://openalex.org/W2490857485","https://openalex.org/W2610478657","https://openalex.org/W2787894218","https://openalex.org/W2948210953","https://openalex.org/W3034730645","https://openalex.org/W3216779397","https://openalex.org/W4205306264","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W4248458271","https://openalex.org/W4210468047","https://openalex.org/W2105845572","https://openalex.org/W2801427017","https://openalex.org/W2804545603","https://openalex.org/W2414642490","https://openalex.org/W4214640225","https://openalex.org/W2099724052","https://openalex.org/W1511070398","https://openalex.org/W2088070094"],"abstract_inverted_index":{"Automated":[0],"essay":[1],"scoring":[2],"engines":[3],"(AESEs)":[4],"are":[5,23,113,131],"becoming":[6],"increasingly":[7],"popular":[8],"as":[9],"an":[10,121],"efficient":[11],"method":[12],"for":[13],"performance":[14],"assessments":[15,21],"in":[16,115,133],"writing,":[17],"including":[18],"many":[19],"language":[20],"that":[22,39,109],"used":[24,30,50,118],"worldwide.":[25],"Before":[26],"they":[27],"can":[28],"be":[29,34],"operationally,":[31],"AESEs":[32,54],"must":[33],"\u201ctrained\u201d":[35],"using":[36],"machine-learning":[37],"techniques":[38],"incorporate":[40],"human":[41,48],"ratings.":[42],"However,":[43],"the":[44,47,53,61,72,94,100,106,116,123],"quality":[45,73,101],"of":[46,63,74,96,102,135],"ratings":[49,117],"to":[51,92,119,140],"train":[52,120],"is":[55,77],"rarely":[56],"examined.":[57],"As":[58],"a":[59,87],"result,":[60],"impact":[62,95],"various":[64],"rater":[65,97,111],"effects":[66,98,112],"(e.g.,":[67],"severity":[68],"and":[69,137],"centrality)":[70],"on":[71,99],"AESE-assigned":[75,103],"scores":[76,125],"not":[78],"known.":[79],"In":[80],"this":[81],"study,":[82],"we":[83],"use":[84],"data":[85],"from":[86],"large-scale":[88],"rater-mediated":[89],"writing":[90],"assessment":[91],"examine":[93],"scores.":[104],"Overall,":[105],"results":[107],"suggest":[108],"if":[110],"present":[114],"AESE,":[122],"AESE":[124],"may":[126],"replicate":[127],"these":[128],"effects.":[129],"Implications":[130],"discussed":[132],"terms":[134],"research":[136],"practice":[138],"related":[139],"automated":[141],"scoring.":[142]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1}],"updated_date":"2026-09-22T06:50:58.402377","created_date":"2017-11-17T00:00:00"}