{"id":"https://openalex.org/W4400773813","doi":"https://doi.org/10.1007/s40593-024-00418-w","title":"Automated Scoring of Constructed Response Items in Math Assessment Using Large Language Models","display_name":"Automated Scoring of Constructed Response Items in Math Assessment Using Large Language Models","publication_year":2024,"publication_date":"2024-07-18","ids":{"openalex":"https://openalex.org/W4400773813","doi":"https://doi.org/10.1007/s40593-024-00418-w"},"language":"en","primary_location":{"id":"doi:10.1007/s40593-024-00418-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00418-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047788899","display_name":"Wesley Morris","orcid":"https://orcid.org/0000-0001-6316-6479"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Wesley Morris","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090861447","display_name":"Langdon Holmes","orcid":"https://orcid.org/0000-0003-4338-4609"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Langdon Holmes","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070821758","display_name":"Joon Suh Choi","orcid":"https://orcid.org/0000-0002-7732-0366"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joon Suh Choi","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028338673","display_name":"Scott A. Crossley","orcid":"https://orcid.org/0000-0002-5148-0273"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Scott Crossley","raw_affiliation_strings":["Vanderbilt University, Nashville, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, USA","institution_ids":["https://openalex.org/I200719446"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5047788899"],"corresponding_institution_ids":["https://openalex.org/I200719446"],"apc_list":{"value":2190,"currency":"EUR","value_usd":2790},"apc_paid":{"value":2190,"currency":"EUR","value_usd":2790},"fwci":5.5572,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.96354339,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"35","issue":"2","first_page":"559","last_page":"586"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9750000238418579,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9563000202178955,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5388708710670471},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.47495293617248535},{"id":"https://openalex.org/keywords/educational-technology","display_name":"Educational technology","score":0.43135982751846313},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.41661933064460754},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34951257705688477},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3430212140083313}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5388708710670471},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.47495293617248535},{"id":"https://openalex.org/C16443162","wikidata":"https://www.wikidata.org/wiki/Q1068473","display_name":"Educational technology","level":2,"score":0.43135982751846313},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41661933064460754},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34951257705688477},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3430212140083313}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s40593-024-00418-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00418-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s40593-024-00418-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00418-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00418-w.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8500000238418579,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400773813.pdf","grobid_xml":"https://content.openalex.org/works/W4400773813.grobid-xml"},"referenced_works_count":62,"referenced_works":["https://openalex.org/W16424739","https://openalex.org/W89404402","https://openalex.org/W188902632","https://openalex.org/W335981425","https://openalex.org/W1614298861","https://openalex.org/W1677486924","https://openalex.org/W1981846272","https://openalex.org/W2026876994","https://openalex.org/W2072773233","https://openalex.org/W2091825929","https://openalex.org/W2107253835","https://openalex.org/W2122490249","https://openalex.org/W2154383645","https://openalex.org/W2166024433","https://openalex.org/W2183293350","https://openalex.org/W2250539671","https://openalex.org/W2292308365","https://openalex.org/W2339946172","https://openalex.org/W2465637239","https://openalex.org/W2513973860","https://openalex.org/W2572698148","https://openalex.org/W2726029799","https://openalex.org/W2770251229","https://openalex.org/W2896457183","https://openalex.org/W2921197258","https://openalex.org/W2921750880","https://openalex.org/W2943565537","https://openalex.org/W2963133108","https://openalex.org/W2981852735","https://openalex.org/W2983149555","https://openalex.org/W3012049595","https://openalex.org/W3016473712","https://openalex.org/W3033187248","https://openalex.org/W3134748792","https://openalex.org/W3134997278","https://openalex.org/W3152268000","https://openalex.org/W3153451655","https://openalex.org/W3157824977","https://openalex.org/W3158196282","https://openalex.org/W3201213805","https://openalex.org/W3211193064","https://openalex.org/W3216581565","https://openalex.org/W4205523551","https://openalex.org/W4212774754","https://openalex.org/W4281384897","https://openalex.org/W4281704958","https://openalex.org/W4283819716","https://openalex.org/W4288059447","https://openalex.org/W4307159619","https://openalex.org/W4313119540","https://openalex.org/W4320731315","https://openalex.org/W4364356639","https://openalex.org/W4382567099","https://openalex.org/W4382567126","https://openalex.org/W4384918448","https://openalex.org/W4385572092","https://openalex.org/W4389518949","https://openalex.org/W4389520742","https://openalex.org/W6675354045","https://openalex.org/W6739901393","https://openalex.org/W6768432047","https://openalex.org/W7026832171"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W1979597421","https://openalex.org/W2007980826","https://openalex.org/W2061531152","https://openalex.org/W3002753104","https://openalex.org/W2077600819","https://openalex.org/W2142036596","https://openalex.org/W2072657027","https://openalex.org/W2600246793","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Abstract":[0],"Recent":[1],"developments":[2],"in":[3,13,21,54,156,193],"the":[4,14,26,38,51,55,72,132,149,157,170,190],"field":[5],"of":[6,17,28,58,97,131,189],"artificial":[7],"intelligence":[8],"allow":[9],"for":[10,25,75,152],"improved":[11],"performance":[12],"automated":[15,159],"assessment":[16],"extended":[18],"response":[19],"items":[20,30,53,192],"mathematics,":[22],"potentially":[23],"allowing":[24],"scoring":[27,161],"these":[29],"cheaply":[31],"and":[32,83,94,126],"at":[33],"scale.":[34],"This":[35,78,173],"study":[36],"details":[37],"grand":[39],"prize-winning":[40],"approach":[41,66,174],"to":[42,48,122,138],"developing":[43],"large":[44,104,145],"language":[45,105,146],"models":[46,147],"(LLMs)":[47],"automatically":[49],"score":[50],"ten":[52,191],"National":[56],"Assessment":[57],"Educational":[59],"Progress":[60],"(NAEP)":[61],"Math":[62],"Scoring":[63],"Challenge.":[64],"The":[65],"uses":[67],"extensive":[68],"preprocessing":[69],"that":[70,119],"balanced":[71],"class":[73],"labels":[74],"each":[76,123,153],"item.":[77],"was":[79],"done":[80],"by":[81],"identifying":[82],"filtering":[84],"over-represented":[85],"classes":[86,99],"using":[87,100],"a":[88,101,194],"classifier":[89],"trained":[90],"on":[91,148,186],"document-term":[92],"matrices":[93],"data":[95],"augmentation":[96],"under-represented":[98],"generative":[102],"pre-trained":[103,144],"model":[106],"(Grammarly\u2019s":[107],"Coedit-XL;":[108],"Raheja":[109],"et":[110,166],"al.,":[111,167],"2023).":[112],"We":[113],"also":[114],"use":[115],"input":[116,151],"modification":[117],"schemes":[118],"were":[120],"hand-crafted":[121],"item":[124,155],"type":[125],"included":[127],"information":[128],"from":[129,183],"parts":[130],"multi-step":[133],"math":[134,160],"problem":[135],"students":[136],"had":[137],"solve.":[139],"Finally,":[140],"we":[141],"finetune":[142],"several":[143],"modified":[150],"individual":[154],"NAEP":[158],"challenge,":[162],"with":[163],"DeBERTa":[164],"(He":[165],"2021a)":[168],"showing":[169],"best":[171],"performance.":[172],"achieved":[175],"human-like":[176],"agreement":[177],"(less":[178],"than":[179],"QWK":[180],"0.05":[181],"difference":[182],"human\u2013human":[184],"agreement)":[185],"nine":[187],"out":[188],"held-out":[195],"test":[196],"set.":[197]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":15}],"updated_date":"2026-04-01T17:29:45.350535","created_date":"2025-10-10T00:00:00"}