{"id":"https://openalex.org/W2016936751","doi":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","title":"AUTOMATIC SCORING OF ADVANCED PLACEMENT BIOLOGY ESSAYS","display_name":"AUTOMATIC SCORING OF ADVANCED PLACEMENT BIOLOGY ESSAYS","publication_year":1997,"publication_date":"1997-12-01","ids":{"openalex":"https://openalex.org/W2016936751","doi":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","mag":"2016936751"},"language":"en","primary_location":{"id":"doi:10.1002/j.2333-8504.1997.tb01743.x","doi":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","is_oa":true,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.1997.tb01743.x","source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W2016936751"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.1997.tb01743.x","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030991878","display_name":"Jill C. Burstein","orcid":"https://orcid.org/0000-0001-7725-7574","observed_orcids":["https://orcid.org/0000-0001-7725-7574"]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jill Burstein","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047979092","display_name":"Randy M. Kaplan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Randy Kaplan","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085277188","display_name":"Susanne Wolff","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Susanne Wolff","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108692383","display_name":"Chi Lu","orcid":"https://orcid.org/0009-0009-1478-0697","observed_orcids":["https://orcid.org/0009-0009-1478-0697"]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chi Lu","raw_affiliation_strings":["Educational Testing Service"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service","institution_ids":["https://openalex.org/I1341030882"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1341030882"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"1997","issue":"2","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.8910999894142151,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.8910999894142151,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.054999999701976776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.008899999782443047,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-essay-scoring","display_name":"automated essay scoring","score":0.8450000286102295},{"id":"https://openalex.org/keywords/human-rater-agreement","display_name":"human-rater agreement","score":0.5569999814033508},{"id":"https://openalex.org/keywords/analytic-rubrics","display_name":"analytic rubrics","score":0.6740000247955322},{"id":"https://openalex.org/keywords/computer-based-testing","display_name":"computer-based testing","score":0.6150000095367432}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9197494983673096},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6573425531387329},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.48979830741882324},{"id":"https://openalex.org/C2984538763","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring system","level":2,"score":0.46345967054367065},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4305614233016968},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4287658929824829},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.41397935152053833},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3561045825481415},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3407638669013977},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2894439101219177},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13854098320007324},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0999341607093811},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.09339821338653564},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1002/j.2333-8504.1997.tb01743.x","doi":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","is_oa":true,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.1997.tb01743.x","source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1002/j.2333-8504.1997.tb01743.x","doi":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","is_oa":true,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.1997.tb01743.x","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.1997.tb01743.x","source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2016936751.pdf","grobid_xml":"https://content.openalex.org/works/W2016936751.grobid-xml"},"referenced_works_count":2,"referenced_works":["https://openalex.org/W1844669744","https://openalex.org/W1940278502"],"related_works":["https://openalex.org/W2137717291","https://openalex.org/W7096386656","https://openalex.org/W2606600550","https://openalex.org/W2177580179","https://openalex.org/W2779520112","https://openalex.org/W1581254716","https://openalex.org/W4206208464","https://openalex.org/W2361950075","https://openalex.org/W3040355829","https://openalex.org/W7217268357"],"abstract_inverted_index":{"ABSTRACT":[0],"The":[1,21,162,191,206],"Natural":[2],"Language":[3],"Processing":[4],"Initiative":[5],"(NLPI)":[6],"at":[7],"ETS":[8],"currently":[9],"does":[10],"research":[11,217],"and":[12,55,99,155,203,213],"development":[13,68,141],"for":[14,31,91,151],"computer\u2010assisted":[15,29],"scoring":[16,32,61,89,149,160,220],"of":[17,23,69,78,158,208,221],"natural":[18,36],"language":[19,37],"constructed\u2010responses.":[20],"purpose":[22],"NLPI's":[24,59],"work":[25],"is":[26],"to":[27,53,66,73,80,117,120,126,130,166],"develop":[28],"methods":[30],"so":[33],"that":[34,169],"computer\u2010administered":[35],"constructed\u2010response":[38],"items":[39],"can":[40,47],"be":[41,48],"integrated":[42],"into":[43],"ETS'":[44],"tests":[45],"which":[46],"scored":[49],"efficiently":[50],"with":[51,223],"regard":[52],"time":[54],"cost.":[56],"Until":[57],"recently,":[58],"automated":[60,219],"efforts":[62],"were":[63],"primarily":[64],"devoted":[65],"the":[67,105,110,140,156,159,195,209],"computer":[70,148],"programs":[71],"used":[72,119,165],"score":[74,121,132,167],"short\u2010answer":[75,122],"responses":[76,123],"constructed\u2010responses":[77],"up":[79],"15":[81],"words.":[82],"In":[83,109],"previous":[84],"studies,":[85],"we":[86,113],"have":[87],"developed":[88],"systems":[90],"main":[92],"idea":[93],"item":[94,102,107],"types":[95,103],"(e.g.,":[96,104],"GUIDES":[97],"items)":[98],"divergent":[100],"thinking":[101],"Formulating\u2010Hypotheses":[106],"(FH)).":[108],"current":[111],"study,":[112],"applied":[114],"similar":[115],"techniques":[116],"those":[118],"in":[124,144,194,218],"order":[125],"implement":[127],"a":[128,146,181],"prototype":[129,150,163],"automatically":[131],"AP":[133,152],"Biology":[134,153],"essays.":[135],"This":[136],"paper":[137,196],"reports":[138],"on":[139],"process":[142],"involved":[143],"building":[145],"linguistically\u2010based,":[147],"essays":[154,168,222],"results":[157,192,207],"system.":[161],"was":[164],"had":[170],"been":[171],"assigned":[172],"an":[173],"Excellent":[174],"rating":[175,183],"(9":[176],"or":[177,180],"10":[178],"points)":[179,187],"Poor":[182],"(0":[184],"\u2010":[185],"3":[186],"by":[188],"human":[189,200],"graders.1":[190],"reported":[193],"illustrate":[197],"agreement":[198],"between":[199],"rater":[201],"scores":[202],"computer\u2010based":[204],"scores.":[205],"study":[210],"are":[211],"promising":[212],"would":[214],"encourage":[215],"continued":[216],"highly":[224],"analytical":[225],"rubrics.":[226]},"counts_by_year":[{"year":2017,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2016-06-24T00:00:00"}