{"id":"https://openalex.org/W1997119722","doi":"https://doi.org/10.2190/cx92-7wkv-n7wc-jl0a","title":"COMPARING THE VALIDITY OF AUTOMATED AND HUMAN SCORING OF ESSAYS","display_name":"COMPARING THE VALIDITY OF AUTOMATED AND HUMAN SCORING OF ESSAYS","alternate_titles":[],"publication_year":2002,"publication_date":"2002-01-01","ids":{"openalex":"https://openalex.org/W1997119722","doi":"https://doi.org/10.2190/cx92-7wkv-n7wc-jl0a","mag":"1997119722"},"language":"en","primary_location":{"id":"doi:10.2190/cx92-7wkv-n7wc-jl0a","doi":"https://doi.org/10.2190/cx92-7wkv-n7wc-jl0a","is_oa":false,"landing_page_url":"https://doi.org/10.2190/cx92-7wkv-n7wc-jl0a","pdf_url":null,"source":{"id":"https://openalex.org/S127795871","display_name":"Journal of Educational Computing Research","issn_l":"0735-6331","issn":["0735-6331","1541-4140"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","erih-plus","jufo-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Educational Computing Research","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W1997119722"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110143742","display_name":"Donald E. Powers","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"DONALD E. POWERS","raw_affiliation_strings":["Educational Testing Service, Princeton, New Jersey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, New Jersey","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030991878","display_name":"Jill C. Burstein","orcid":"https://orcid.org/0000-0001-7725-7574","observed_orcids":["https://orcid.org/0000-0001-7725-7574"]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"JILL C. BURSTEIN","raw_affiliation_strings":["Educational Testing Service, Princeton, New Jersey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, New Jersey","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041834124","display_name":"Martin S. Chodorow","orcid":"https://orcid.org/0000-0001-5780-2598","observed_orcids":["https://orcid.org/0000-0001-5780-2598"]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"MARTIN S. CHODOROW","raw_affiliation_strings":["Educational Testing Service, Princeton, New Jersey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, New Jersey","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042657212","display_name":"Mary E. Fowles","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"MARY E. FOWLES","raw_affiliation_strings":["Educational Testing Service, Princeton, New Jersey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, New Jersey","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056855678","display_name":"Karen Kukich","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"KAREN KUKICH","raw_affiliation_strings":["Educational Testing Service, Princeton, New Jersey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, New Jersey","institution_ids":["https://openalex.org/I1341030882"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1341030882"],"apc_list":{"value":3900,"currency":"USD","value_usd":3900},"apc_paid":null,"fwci":2.0103,"has_fulltext":false,"cited_by_count":56,"citation_normalized_percentile":{"value":0.88650832,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":98},"biblio":{"volume":"26","issue":"4","first_page":"407","last_page":"425"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.7002000212669373,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.7002000212669373,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.1152999997138977,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.044599998742341995,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-essay-scoring","display_name":"automated essay scoring","score":0.9150000214576721},{"id":"https://openalex.org/keywords/human-scoring","display_name":"human scoring","score":0.8130000233650208},{"id":"https://openalex.org/keywords/scoring-validity","display_name":"scoring validity","score":0.6499999761581421},{"id":"https://openalex.org/keywords/writing-skills","display_name":"writing skills","score":0.7720000147819519},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.8550000190734863},{"id":"https://openalex.org/keywords/computer-based-testing","display_name":"computer-based testing","score":0.5130000114440918}],"concepts":[{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6479735374450684},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6194759607315063},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6184260249137878},{"id":"https://openalex.org/C2777898490","wikidata":"https://www.wikidata.org/wiki/Q17157236","display_name":"Writing assessment","level":2,"score":0.544115424156189},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4707518219947815},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4175660014152527},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4062202572822571},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3603026270866394},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.32293185591697693},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.2727838158607483},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.2190/cx92-7wkv-n7wc-jl0a","doi":"https://doi.org/10.2190/cx92-7wkv-n7wc-jl0a","is_oa":false,"landing_page_url":"https://doi.org/10.2190/cx92-7wkv-n7wc-jl0a","pdf_url":null,"source":{"id":"https://openalex.org/S127795871","display_name":"Journal of Educational Computing Research","issn_l":"0735-6331","issn":["0735-6331","1541-4140"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","erih-plus","jufo-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Educational Computing Research","raw_type":"journal-article"},{"id":"doi:10.1092/up3h-m3te-q290-qj2t","doi":"https://doi.org/10.1092/up3h-m3te-q290-qj2t","is_oa":false,"landing_page_url":"https://doi.org/10.1092/up3h-m3te-q290-qj2t","pdf_url":null,"source":{"id":"https://openalex.org/S127795871","display_name":"Journal of Educational Computing Research","issn_l":"0735-6331","issn":["0735-6331","1541-4140"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","erih-plus","jufo-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Educational Computing Research","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.6164000034332275}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W190485413","https://openalex.org/W602487103","https://openalex.org/W1553202205","https://openalex.org/W1590881479","https://openalex.org/W1979342075","https://openalex.org/W2015391954","https://openalex.org/W2016195898","https://openalex.org/W2018072436","https://openalex.org/W2028829385","https://openalex.org/W2028856469","https://openalex.org/W2029299436","https://openalex.org/W2029961360","https://openalex.org/W2046556834","https://openalex.org/W2059677116","https://openalex.org/W2094401778","https://openalex.org/W2105204564","https://openalex.org/W2107484888","https://openalex.org/W2143834603","https://openalex.org/W2154902941","https://openalex.org/W2155652017","https://openalex.org/W2158032146","https://openalex.org/W2158997610","https://openalex.org/W2166708896","https://openalex.org/W2725968356","https://openalex.org/W2796807530","https://openalex.org/W3034730645","https://openalex.org/W4213276201","https://openalex.org/W4249954285","https://openalex.org/W4391464594"],"related_works":["https://openalex.org/W2046517742","https://openalex.org/W2404547900","https://openalex.org/W6912898212","https://openalex.org/W3163354334","https://openalex.org/W2105845572","https://openalex.org/W4361286724","https://openalex.org/W745077393","https://openalex.org/W4236730842","https://openalex.org/W2551276343","https://openalex.org/W1503127045"],"abstract_inverted_index":{"Automated,":[0],"or":[1],"computer-based,":[2],"scoring":[3,58,69,171],"represents":[4],"one":[5,222],"promising":[6],"possibility":[7],"for":[8,132,136,201,205,225],"improving":[9],"the":[10,56,65,79,98,113,155,163,167,185,191,213,219],"cost":[11],"effectiveness":[12],"(and":[13],"other":[14],"features)":[15],"of":[16,24,40,59,64,67,100,106,115,128,162,170,175,190,194,218,221],"complex":[17],"performance":[18],"assessments":[19],"(such":[20],"as":[21,160],"direct":[22],"tests":[23],"writing":[25,108,121,130,176],"skill)":[26],"that":[27],"require":[28],"examinees":[29],"to":[30,55,96,103,123,145,165],"construct":[31],"responses":[32],"rather":[33],"than":[34,204],"select":[35],"them":[36],"from":[37,117],"a":[38,118],"set":[39],"multiple":[41],"choices.":[42],"Indeed,":[43],"significant":[44],"advances":[45],"have":[46],"been":[47,94],"made":[48],"in":[49],"applying":[50],"natural":[51],"language":[52],"processing":[53],"techniques":[54],"automatic":[57],"essays.":[60],"Thus":[61],"far,":[62],"most":[63],"validation":[66],"automated":[68,101,133,147,202,226],"has":[70,93],"focused":[71],"appropriately":[72],"(but":[73],"too":[74],"narrowly,":[75],"we":[76],"contend)":[77],"on":[78],"correspondence":[80],"between":[81,184],"computer-generated":[82],"scores":[83,102,116,134,137,150,203,206],"and":[84,135,148,188],"those":[85],"assigned":[86,138],"by":[87,139,208],"human":[88,141,149,209],"readers.":[89,142,210],"Far":[90],"less":[91],"effort":[92],"devoted":[95],"assessing":[97],"relation":[99],"independent":[104],"indicators":[105,127,157,187],"examinees'":[107,129],"skills.":[109],"This":[110],"study":[111],"examined":[112],"relationship":[114],"graduate":[119],"level":[120],"assessment":[122],"several":[124],"independent,":[125],"non-test":[126,156,186],"skills\u2014both":[131],"trained":[140],"The":[143],"extent":[144],"which":[146,166],"exhibited":[151],"similar":[152,173],"relations":[153,197],"with":[154],"was":[158],"taken":[159],"evidence":[161,217],"degree":[164],"two":[168,192],"methods":[169,193],"reflect":[172],"aspects":[174],"proficiency.":[177],"Analyses":[178],"revealed":[179],"significant,":[180],"but":[181],"modest,":[182],"correlations":[183],"each":[189],"scoring.":[195,227],"These":[196],"were":[198],"somewhat":[199],"weaker":[200],"awarded":[207],"Overall,":[211],"however,":[212],"results":[214],"provide":[215],"some":[216],"validity":[220],"specific":[223],"procedure":[224]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W1997119722","counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":2},{"year":2012,"cited_by_count":5}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2016-06-24T00:00:00"}
