{"id":"https://openalex.org/W1998297147","doi":"https://doi.org/10.1002/j.2333-8504.2007.tb02063.x","title":"CONSTRUCT VALIDITY OF E\u2010RATER\u00ae IN SCORING TOEFL\u00ae ESSAYS","display_name":"CONSTRUCT VALIDITY OF E\u2010RATER\u00ae IN SCORING TOEFL\u00ae ESSAYS","publication_year":2007,"publication_date":"2007-06-01","ids":{"openalex":"https://openalex.org/W1998297147","doi":"https://doi.org/10.1002/j.2333-8504.2007.tb02063.x","mag":"1998297147"},"language":"en","primary_location":{"id":"doi:10.1002/j.2333-8504.2007.tb02063.x","doi":"https://doi.org/10.1002/j.2333-8504.2007.tb02063.x","is_oa":false,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.2007.tb02063.x","pdf_url":null,"source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043400579","display_name":"Yigal Attali","orcid":"https://orcid.org/0000-0002-3907-247X","observed_orcids":["https://orcid.org/0000-0002-3907-247X"]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yigal Attali","raw_affiliation_strings":["ETS  Princeton NJ","ETS, Princeton, NJ#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETS  Princeton NJ","institution_ids":["https://openalex.org/I1341030882"]},{"raw_affiliation_string":"ETS, Princeton, NJ#TAB#","institution_ids":["https://openalex.org/I1341030882"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5043400579"],"corresponding_institution_ids":["https://openalex.org/I1341030882"],"apc_list":null,"apc_paid":null,"fwci":1.7994,"has_fulltext":false,"cited_by_count":46,"citation_normalized_percentile":{"value":0.85727436,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":"2007","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.5752000212669373,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.5752000212669373,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.2240000069141388,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.10909999907016754,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/e-rater","display_name":"e-rater","score":0.7900000214576721},{"id":"https://openalex.org/keywords/automated-essay-scoring","display_name":"automated essay scoring","score":0.9729999899864197},{"id":"https://openalex.org/keywords/construct-validity","display_name":"construct validity","score":0.8019999861717224},{"id":"https://openalex.org/keywords/toefl","display_name":"TOEFL","score":0.8410000205039978},{"id":"https://openalex.org/keywords/essay-writing","display_name":"essay writing","score":0.5910000205039978},{"id":"https://openalex.org/keywords/score-reliability","display_name":"scoring reliability","score":0.671999990940094},{"id":"https://openalex.org/keywords/essay-length","display_name":"essay length","score":0.7639999985694885},{"id":"https://openalex.org/keywords/country-of-origin-differences","display_name":"country of origin differences","score":0.546999990940094},{"id":"https://openalex.org/keywords/word-usage","display_name":"word usage","score":0.7549999952316284},{"id":"https://openalex.org/keywords/grammar","display_name":"grammar","score":0.9909999966621399},{"id":"https://openalex.org/keywords/discourse","display_name":"discourse","score":0.9240000247955322}],"concepts":[{"id":"https://openalex.org/C18563106","wikidata":"https://www.wikidata.org/wiki/Q487425","display_name":"Test of English as a Foreign Language","level":3,"score":0.7206928133964539},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.7192710041999817},{"id":"https://openalex.org/C49453240","wikidata":"https://www.wikidata.org/wiki/Q1592163","display_name":"Construct validity","level":3,"score":0.5937077403068542},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.5128012299537659},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.5111895203590393},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4922923147678375},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.49013957381248474},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4766564965248108},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4739494323730469},{"id":"https://openalex.org/C116211729","wikidata":"https://www.wikidata.org/wiki/Q5372350","display_name":"Test validity","level":3,"score":0.4238618016242981},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.3368440866470337},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.3038730025291443},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.29039329290390015},{"id":"https://openalex.org/C166007726","wikidata":"https://www.wikidata.org/wiki/Q4536542","display_name":"Language assessment","level":2,"score":0.22537419199943542},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.20330193638801575},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.1986989974975586},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14138245582580566},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.10161471366882324},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1002/j.2333-8504.2007.tb02063.x","doi":"https://doi.org/10.1002/j.2333-8504.2007.tb02063.x","is_oa":false,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.2007.tb02063.x","pdf_url":null,"source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6164000034332275,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W31310150","https://openalex.org/W1732149515","https://openalex.org/W1954968711","https://openalex.org/W1993225241","https://openalex.org/W1997119722","https://openalex.org/W2001402876","https://openalex.org/W2003192467","https://openalex.org/W2020769785","https://openalex.org/W2034131105","https://openalex.org/W2088696944","https://openalex.org/W2090412809","https://openalex.org/W2094401778","https://openalex.org/W2105204564","https://openalex.org/W2154414741","https://openalex.org/W2158240052","https://openalex.org/W2158997610","https://openalex.org/W2166708896","https://openalex.org/W2184872199","https://openalex.org/W2905010601","https://openalex.org/W2990534221","https://openalex.org/W3034730645","https://openalex.org/W4213276201","https://openalex.org/W4285719527","https://openalex.org/W6683687954"],"related_works":["https://openalex.org/W2135972341","https://openalex.org/W2037930515","https://openalex.org/W2586377873","https://openalex.org/W2967146216","https://openalex.org/W1993667300","https://openalex.org/W2088696944","https://openalex.org/W1990278592","https://openalex.org/W3197482157","https://openalex.org/W6990111457","https://openalex.org/W2068099813"],"abstract_inverted_index":{"ABSTRACT":[0],"This":[1],"study":[2],"examined":[3,111],"the":[4,8,21,37,52,58,63,70,79,135,140,150,162,169,174,198,214,227],"construct":[5,176],"validity":[6],"of":[7,23,33,73,83,120,142,149],"e\u2010rater\u00ae":[9],"automated":[10],"essay":[11,25,60,101,179,207],"scoring":[12,19],"engine":[13],"as":[14,177],"an":[15],"alternative":[16],"to":[17,114,138],"human":[18,59,84,106,170,236],"in":[20,49,116,124,226],"context":[22],"TOEFL\u00ae":[24],"writing.":[26],"Analyses":[27],"were":[28,47,88,110],"based":[29,54,65,165],"on":[30,55,66,134,166],"a":[31,40,76,129],"sample":[32],"students":[34],"who":[35],"repeated":[36],"TOEFL":[38,92],"within":[39],"short":[41],"time":[42],"period.":[43],"Two":[44],"e\u2010rater":[45,86,108,136,152,163,195,231],"scores":[46,87,109,153,180,232],"investigated":[48],"this":[50,155],"study,":[51],"first":[53],"optimally":[56,167],"predicting":[57,168],"score":[61,164,171,196,237],"and":[62,81,85,97,99,107,122,146,185,220,241],"second":[64],"equal":[67],"weights":[68],"for":[69],"different":[71],"features":[72,137],"e\u2010rater.":[74],"Within":[75],"multitrait\u2010multimethod":[77],"approach,":[78],"correlations":[80,188],"reliabilities":[82],"analyzed":[89],"together":[90],"with":[91,100,112,181,189,206,213],"subscores":[93],"(structured":[94],"writing,":[95],"reading,":[96],"listening)":[98],"length.":[102,208],"Possible":[103],"biases":[104],"between":[105,239],"respect":[113],"differences":[115,123,238],"performance":[117],"across":[118,126],"countries":[119,240],"origin":[121],"difficulty":[125],"prompts.":[127,242],"Finally,":[128],"factor":[130,228],"analysis":[131],"was":[132,224],"conducted":[133],"investigate":[139],"interpretability":[141],"their":[143],"internal":[144],"structure":[145,156,222],"determine":[147],"which":[148],"two":[151],"reflects":[154],"more":[157],"closely.":[158],"Results":[159],"showed":[160,197],"that":[161,223],"measures":[172],"essentially":[173],"same":[175,199],"human\u2010based":[178],"significantly":[182,203],"higher":[183,187],"reliability":[184,201],"consequently":[186],"related":[190],"language":[191],"scores.":[192],"The":[193],"equal\u2010weights":[194],"high":[200],"but":[202],"lower":[204],"correlation":[205],"It":[209],"is":[210],"also":[211,233],"aligned":[212],"3\u2010factor":[215],"hierarchical":[216],"(word":[217],"use,":[218],"grammar,":[219],"discourse)":[221],"discovered":[225],"analysis.":[229],"Both":[230],"successfully":[234],"replicate":[235]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":1},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":5},{"year":2013,"cited_by_count":4},{"year":2012,"cited_by_count":4}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}