{"id":"https://openalex.org/W4400773813","doi":"https://doi.org/10.1007/s40593-024-00418-w","title":"Automated Scoring of Constructed Response Items in Math Assessment Using Large Language Models","display_name":"Automated Scoring of Constructed Response Items in Math Assessment Using Large Language Models","publication_year":2024,"publication_date":"2024-07-18","ids":{"openalex":"https://openalex.org/W4400773813","doi":"https://doi.org/10.1007/s40593-024-00418-w"},"language":"en","primary_location":{"id":"doi:10.1007/s40593-024-00418-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00418-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047788899","display_name":"Wesley Morris","orcid":"https://orcid.org/0000-0001-6316-6479"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Wesley Morris","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":"https://orcid.org/0000-0001-6316-6479","affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090861447","display_name":"Langdon Holmes","orcid":"https://orcid.org/0000-0003-4338-4609"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Langdon Holmes","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070821758","display_name":"Joon Suh Choi","orcid":"https://orcid.org/0000-0002-7732-0366"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joon Suh Choi","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028338673","display_name":"Scott A. Crossley","orcid":"https://orcid.org/0000-0002-5148-0273"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Scott Crossley","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5047788899"],"corresponding_institution_ids":["https://openalex.org/I200719446"],"apc_list":{"value":3190,"currency":"USD","value_usd":3190},"apc_paid":{"value":3190,"currency":"USD","value_usd":3190},"fwci":6.7593,"has_fulltext":true,"cited_by_count":29,"citation_normalized_percentile":{"value":0.97228593,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"35","issue":"2","first_page":"559","last_page":"586"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9750000238418579,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9563000202178955,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5388708710670471},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.47495293617248535},{"id":"https://openalex.org/keywords/educational-technology","display_name":"Educational technology","score":0.43135982751846313},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.41661933064460754},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34951257705688477},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3430212140083313}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5388708710670471},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.47495293617248535},{"id":"https://openalex.org/C16443162","wikidata":"https://www.wikidata.org/wiki/Q1068473","display_name":"Educational technology","level":2,"score":0.43135982751846313},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41661933064460754},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34951257705688477},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3430212140083313}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s40593-024-00418-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00418-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s40593-024-00418-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00418-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8500000238418579}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400773813.pdf","grobid_xml":"https://content.openalex.org/works/W4400773813.grobid-xml"},"referenced_works_count":62,"referenced_works":["https://openalex.org/W16424739","https://openalex.org/W89404402","https://openalex.org/W188902632","https://openalex.org/W335981425","https://openalex.org/W1614298861","https://openalex.org/W1677486924","https://openalex.org/W1981846272","https://openalex.org/W2026876994","https://openalex.org/W2072773233","https://openalex.org/W2091825929","https://openalex.org/W2107253835","https://openalex.org/W2122490249","https://openalex.org/W2154383645","https://openalex.org/W2166024433","https://openalex.org/W2183293350","https://openalex.org/W2250539671","https://openalex.org/W2292308365","https://openalex.org/W2339946172","https://openalex.org/W2465637239","https://openalex.org/W2513973860","https://openalex.org/W2572698148","https://openalex.org/W2726029799","https://openalex.org/W2770251229","https://openalex.org/W2896457183","https://openalex.org/W2921197258","https://openalex.org/W2921750880","https://openalex.org/W2943565537","https://openalex.org/W2963133108","https://openalex.org/W2981852735","https://openalex.org/W2983149555","https://openalex.org/W3012049595","https://openalex.org/W3016473712","https://openalex.org/W3033187248","https://openalex.org/W3134748792","https://openalex.org/W3134997278","https://openalex.org/W3152268000","https://openalex.org/W3153451655","https://openalex.org/W3157824977","https://openalex.org/W3158196282","https://openalex.org/W3201213805","https://openalex.org/W3211193064","https://openalex.org/W3216581565","https://openalex.org/W4205523551","https://openalex.org/W4212774754","https://openalex.org/W4281384897","https://openalex.org/W4281704958","https://openalex.org/W4283819716","https://openalex.org/W4288059447","https://openalex.org/W4307159619","https://openalex.org/W4313119540","https://openalex.org/W4320731315","https://openalex.org/W4364356639","https://openalex.org/W4382567099","https://openalex.org/W4382567126","https://openalex.org/W4384918448","https://openalex.org/W4385572092","https://openalex.org/W4389518949","https://openalex.org/W4389520742","https://openalex.org/W6675354045","https://openalex.org/W6739901393","https://openalex.org/W6768432047","https://openalex.org/W7026832171"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W1979597421","https://openalex.org/W2007980826","https://openalex.org/W2061531152","https://openalex.org/W3002753104","https://openalex.org/W2077600819","https://openalex.org/W2142036596","https://openalex.org/W2072657027","https://openalex.org/W2600246793","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Recent":[0],"developments":[1],"in":[2,12,20,53,156,194],"the":[3,13,25,37,50,54,71,132,149,157,171,191],"field":[4],"of":[5,16,27,57,96,131,190],"artificial":[6],"intelligence":[7],"allow":[8],"for":[9,24,74,152],"improved":[10],"performance":[11],"automated":[14,159],"assessment":[15],"extended":[17],"response":[18],"items":[19,29,52,193],"mathematics,":[21],"potentially":[22],"allowing":[23],"scoring":[26,161],"these":[28],"cheaply":[30],"and":[31,82,93,126],"at":[32],"scale.":[33],"This":[34,77,174],"study":[35],"details":[36],"grand":[38],"prize-winning":[39],"approach":[40,65,175],"to":[41,47,122,138],"developing":[42],"large":[43,103,145],"language":[44,104,146],"models":[45,147],"(LLMs)":[46],"automatically":[48],"score":[49],"ten":[51,192],"National":[55],"Assessment":[56],"Educational":[58],"Progress":[59],"(NAEP)":[60],"Math":[61],"Scoring":[62],"Challenge.":[63],"The":[64],"uses":[66],"extensive":[67],"preprocessing":[68],"that":[69,119],"balanced":[70],"class":[72],"labels":[73],"each":[75,123,153],"item.":[76],"was":[78],"done":[79],"by":[80],"identifying":[81],"filtering":[83],"over-represented":[84],"classes":[85,98],"using":[86,99],"a":[87,100,195],"classifier":[88],"trained":[89],"on":[90,148,187],"document-term":[91],"matrices":[92],"data":[94],"augmentation":[95],"under-represented":[97],"generative":[101],"pre-trained":[102,144],"model":[105],"(Grammarly\u2019s":[106],"Coedit-XL;":[107],"Raheja":[108],"et":[109,166],"al.,":[110,167],"2023":[111],").":[112],"We":[113],"also":[114],"use":[115],"input":[116,151],"modification":[117],"schemes":[118],"were":[120],"hand-crafted":[121],"item":[124,155],"type":[125],"included":[127],"information":[128],"from":[129,184],"parts":[130],"multi-step":[133],"math":[134,160],"problem":[135],"students":[136],"had":[137],"solve.":[139],"Finally,":[140],"we":[141],"finetune":[142],"several":[143],"modified":[150],"individual":[154],"NAEP":[158],"challenge,":[162],"with":[163],"DeBERTa":[164],"(He":[165],"2021a":[168],")":[169],"showing":[170],"best":[172],"performance.":[173],"achieved":[176],"human-like":[177],"agreement":[178],"(less":[179],"than":[180],"QWK":[181],"0.05":[182],"difference":[183],"human\u2013human":[185],"agreement)":[186],"nine":[188],"out":[189],"held-out":[196],"test":[197],"set.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":15}],"updated_date":"2026-08-25T07:29:55.448023","created_date":"2025-10-10T00:00:00"}