{"id":"https://openalex.org/W4400668042","doi":"https://doi.org/10.1145/3657604.3664693","title":"Can Large Language Models Make the Grade? An Empirical Study Evaluating LLMs Ability To Mark Short Answer Questions in K-12 Education","display_name":"Can Large Language Models Make the Grade? An Empirical Study Evaluating LLMs Ability To Mark Short Answer Questions in K-12 Education","publication_year":2024,"publication_date":"2024-07-09","ids":{"openalex":"https://openalex.org/W4400668042","doi":"https://doi.org/10.1145/3657604.3664693"},"language":"en","primary_location":{"id":"doi:10.1145/3657604.3664693","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3657604.3664693","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eleventh ACM Conference on Learning @ Scale","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092492137","display_name":"Owen Henkel","orcid":null},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Owen Henkel","raw_affiliation_strings":["University of Oxford, Oxford, United Kingdom"],"raw_orcid":"https://orcid.org/0009-0001-8850-067X","affiliations":[{"raw_affiliation_string":"University of Oxford, Oxford, United Kingdom","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017264349","display_name":"Libby Hills","orcid":"https://orcid.org/0009-0004-7392-4711"},"institutions":[{"id":"https://openalex.org/I4210133674","display_name":"Jacobs Foundation","ror":"https://ror.org/03tj32a09","country_code":"CH","type":"other","lineage":["https://openalex.org/I4210133674"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Libby Hills","raw_affiliation_strings":["Jacobs Foundation, Zurich, Switzerland"],"raw_orcid":"https://orcid.org/0009-0004-7392-4711","affiliations":[{"raw_affiliation_string":"Jacobs Foundation, Zurich, Switzerland","institution_ids":["https://openalex.org/I4210133674"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5096886060","display_name":"Adam Boxer","orcid":"https://orcid.org/0009-0005-9524-8037"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Adam Boxer","raw_affiliation_strings":["Carousel Learning, London, United Kingdom"],"raw_orcid":"https://orcid.org/0009-0005-9524-8037","affiliations":[{"raw_affiliation_string":"Carousel Learning, London, United Kingdom","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006622944","display_name":"Bill Roberts","orcid":"https://orcid.org/0009-0005-0508-1895"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bill Roberts","raw_affiliation_strings":["Legible Labs, New York, USA"],"raw_orcid":"https://orcid.org/0009-0005-0508-1895","affiliations":[{"raw_affiliation_string":"Legible Labs, New York, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078797064","display_name":"Zachary Levonian","orcid":"https://orcid.org/0000-0002-8932-1489"},"institutions":[{"id":"https://openalex.org/I4210106943","display_name":"Digital Harbor Foundation","ror":"https://ror.org/02dpw9611","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106943"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zach Levonian","raw_affiliation_strings":["Digital Harbor Foundation, Baltimore, USA"],"raw_orcid":"https://orcid.org/0000-0002-8932-1489","affiliations":[{"raw_affiliation_string":"Digital Harbor Foundation, Baltimore, USA","institution_ids":["https://openalex.org/I4210106943"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":12.7539,"has_fulltext":false,"cited_by_count":39,"citation_normalized_percentile":{"value":0.99217271,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"300","last_page":"304"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12884","display_name":"Educational Assessment and Pedagogy","score":0.9860000014305115,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5950318574905396},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.525770902633667},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.44200369715690613},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33652162551879883},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2425977885723114},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1719319224357605},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12850579619407654}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5950318574905396},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.525770902633667},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.44200369715690613},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33652162551879883},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2425977885723114},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1719319224357605},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12850579619407654}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3657604.3664693","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3657604.3664693","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Eleventh ACM Conference on Learning @ Scale","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8899999856948853,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W1523363798","https://openalex.org/W1549788644","https://openalex.org/W2014220607","https://openalex.org/W2056874695","https://openalex.org/W2076898331","https://openalex.org/W2076929833","https://openalex.org/W2135995390","https://openalex.org/W2147514634","https://openalex.org/W2153022270","https://openalex.org/W2892211760","https://openalex.org/W2953001123","https://openalex.org/W2985449513","https://openalex.org/W3030163527","https://openalex.org/W3037207300","https://openalex.org/W3164972323","https://openalex.org/W4281384897","https://openalex.org/W4320731315","https://openalex.org/W4384662964","https://openalex.org/W4388735460"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"This":[0,96],"paper":[1],"presents":[2],"reports":[3],"on":[4,99],"a":[5,10,67,73,113,129,141],"series":[6],"of":[7,38,119,131],"experiments":[8],"with":[9,80],"novel":[11],"dataset":[12,70],"evaluating":[13],"how":[14,34],"well":[15,35,85],"Large":[16],"Language":[17],"Models":[18],"(LLMs)":[19],"can":[20],"mark":[21],"(i.e.":[22],"grade)":[23],"open":[24],"text":[25],"responses":[26],"to":[27,51,92,117,125],"short":[28,52,107],"answer":[29,53,108],"questions,":[30],"Specifically,":[31],"we":[32],"explore":[33],"different":[36,55],"combinations":[37],"GPT":[39],"version":[40],"and":[41,59,61,133,153],"prompt":[42],"engineering":[43],"strategies":[44],"performed":[45,84],"at":[46,112],"marking":[47],"real":[48],"student":[49],"answers":[50],"across":[54,128],"domain":[56],"areas":[57],"(Science":[58],"History)":[60],"grade-levels":[62],"(spanning":[63],"ages":[64],"5-16)":[65],"using":[66],"new,":[68],"never-used-before":[69],"from":[71],"Carousel,":[72],"quizzing":[74],"platform.":[75],"We":[76],"found":[77],"that":[78,102,118,137],"GPT-4,":[79],"basic":[81],"few-shot":[82],"prompting":[83],"(Kappa,":[86],"0.70)":[87],"and,":[88],"importantly,":[89],"very":[90,115],"close":[91,116],"human-level":[93,126],"performance":[94],"(0.75).":[95],"research":[97],"builds":[98],"prior":[100],"findings":[101],"GPT-4":[103],"could":[104,139],"reliably":[105],"score":[106],"reading":[109],"comprehension":[110],"questions":[111],"performance-level":[114],"expert":[120],"human":[121],"raters.":[122],"The":[123],"proximity":[124],"performance,":[127],"variety":[130],"subjects":[132],"grade":[134],"levels":[135],"suggests":[136],"LLMs":[138],"be":[140],"valuable":[142],"tool":[143],"for":[144,157],"supporting":[145],"low-stakes":[146],"formative":[147],"assessment":[148],"tasks":[149],"in":[150],"K-12":[151],"education":[152,159],"has":[154],"important":[155],"implications":[156],"real-world":[158],"delivery.":[160]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":24},{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}