{"id":"https://openalex.org/W4407842016","doi":"https://doi.org/10.1145/3706468.3706527","title":"Can AI grade your essays? A comparative analysis of large language models and teacher ratings in multidimensional essay scoring","display_name":"Can AI grade your essays? A comparative analysis of large language models and teacher ratings in multidimensional essay scoring","publication_year":2025,"publication_date":"2025-02-21","ids":{"openalex":"https://openalex.org/W4407842016","doi":"https://doi.org/10.1145/3706468.3706527"},"language":"en","primary_location":{"id":"doi:10.1145/3706468.3706527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706527","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3706468.3706527","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013346660","display_name":"Kathrin Se\u00dfler","orcid":"https://orcid.org/0000-0002-3380-4641"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Kathrin Se\u00dfler","raw_affiliation_strings":["Technical University of Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114987805","display_name":"Maurice F\u00fcrstenberg","orcid":"https://orcid.org/0009-0001-1090-9299"},"institutions":[{"id":"https://openalex.org/I8087733","display_name":"University of T\u00fcbingen","ror":"https://ror.org/03a1kwz48","country_code":"DE","type":"education","lineage":["https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Maurice F\u00fcrstenberg","raw_affiliation_strings":["University of T\u00fcbingen, T\u00fcbingen, Germany"],"affiliations":[{"raw_affiliation_string":"University of T\u00fcbingen, T\u00fcbingen, Germany","institution_ids":["https://openalex.org/I8087733"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057566592","display_name":"Babette B\u00fchler","orcid":"https://orcid.org/0000-0003-1679-4979"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Babette B\u00fchler","raw_affiliation_strings":["Technical University of Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008809634","display_name":"Enkelejda Kasneci","orcid":"https://orcid.org/0000-0003-3146-4484"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Enkelejda Kasneci","raw_affiliation_strings":["Technical University of Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5013346660"],"corresponding_institution_ids":["https://openalex.org/I62916508"],"apc_list":null,"apc_paid":null,"fwci":71.5503,"has_fulltext":true,"cited_by_count":30,"citation_normalized_percentile":{"value":0.99906834,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"462","last_page":"472"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.988099992275238,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9865000247955322,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6165816783905029},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5427610278129578},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4686267673969269},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.3747377395629883},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.29604315757751465}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6165816783905029},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5427610278129578},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4686267673969269},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3747377395629883},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.29604315757751465}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3706468.3706527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706527","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:epub.ub.uni-muenchen.de:126938","is_oa":true,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bvb:19-epub-126938-0","pdf_url":"https://epub.ub.uni-muenchen.de/126938/1/Paper_in_Proceedings.pdf","source":{"id":"https://openalex.org/S4306401845","display_name":"Open access LMU (Ludwid Maxmilian's Universitat Munchen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I8204097","host_organization_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","host_organization_lineage":["https://openalex.org/I8204097"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"NonPeerReviewed"}],"best_oa_location":{"id":"doi:10.1145/3706468.3706527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706527","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8799999952316284,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W2055011070","https://openalex.org/W2106695994","https://openalex.org/W2141960018","https://openalex.org/W2327037637","https://openalex.org/W2506826885","https://openalex.org/W2952164904","https://openalex.org/W2964413085","https://openalex.org/W3114208363","https://openalex.org/W3144434203","https://openalex.org/W3170037077","https://openalex.org/W3196589716","https://openalex.org/W3201077663","https://openalex.org/W4229456833","https://openalex.org/W4310123629","https://openalex.org/W4310436400","https://openalex.org/W4323655724","https://openalex.org/W4366403195","https://openalex.org/W4380320806","https://openalex.org/W4382338912","https://openalex.org/W4384918448","https://openalex.org/W4385571232","https://openalex.org/W4385572239","https://openalex.org/W4386261929","https://openalex.org/W4387007101","https://openalex.org/W4387824566","https://openalex.org/W4389344855","https://openalex.org/W4391026420","https://openalex.org/W4392202731","https://openalex.org/W4393011212","https://openalex.org/W4402819599","https://openalex.org/W4406232026","https://openalex.org/W6908570072"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"The":[0,122],"manual":[1],"assessment":[2,157],"and":[3,39,44,77,90,112,140,186],"grading":[4],"of":[5,41,57,70,144,171],"student":[6,50],"writing":[7],"is":[8],"a":[9,160],"time-consuming":[10],"yet":[11],"critical":[12],"task":[13],"for":[14,28],"teachers.":[15,29],"Recent":[16],"developments":[17],"in":[18,47,108,119,136],"generative":[19],"AI":[20],"offer":[21],"potential":[22],"solutions":[23],"to":[24,55,94,163,176,181,184,190],"facilitate":[25],"essay-scoring":[26],"tasks":[27],"In":[30],"our":[31],"study,":[32],"we":[33],"evaluate":[34],"the":[35,137,169,192,195],"performance":[36],"(e.g.":[37],"alignment":[38,113],"reliability)":[40],"both":[42,109],"open-source":[43,106],"closed-source":[45],"LLMs":[46],"assessing":[48],"German":[49],"essays,":[51,172],"comparing":[52],"their":[53,182,187],"evaluations":[54],"those":[56],"37":[58],"teachers":[59],"across":[60],"10":[61],"pre-defined":[62],"criteria":[63],"(i.e.,":[64],"plot":[65],"logic,":[66],"expression).":[67],"A":[68],"corpus":[69],"20":[71],"real-world":[72],"essays":[73],"from":[74],"Year":[75],"7":[76],"8":[78],"students":[79],"was":[80],"analyzed":[81],"using":[82],"five":[83],"LLMs:":[84],"GPT-3.5,":[85],"GPT-4,":[86],"o1-preview,":[87],"LLaMA":[88],"3-70B,":[89],"Mixtral":[91],"8x7B,":[92],"aiming":[93],"provide":[95],"in-depth":[96],"insights":[97],"into":[98],"LLMs\u2019":[99],"scoring":[100],"capabilities.":[101],"Closed-source":[102],"GPT":[103],"models":[104,107,196],"outperform":[105],"internal":[110,142],"consistency":[111,143],"with":[114,133,174],"human":[115,134],"ratings,":[116],"particularly":[117],"excelling":[118],"language-related":[120,177],"criteria.":[121,178],"o1":[123],"model":[124],"outperforms":[125],"all":[126],"other":[127],"LLMs,":[128],"achieving":[129],"Spearman\u2019s":[130],"=":[131,145],".74":[132],"assessments":[135],"Overall":[138],"score,":[139],"an":[141],".80,":[146],"though":[147],"biased":[148],"towards":[149],"higher":[150],"scores.":[151],"These":[152],"findings":[153],"indicate":[154],"that":[155],"LLM-based":[156],"can":[158],"be":[159],"useful":[161],"tool":[162],"reduce":[164],"teacher":[165],"workload":[166],"by":[167],"supporting":[168],"evaluation":[170],"especially":[173],"regard":[175],"However,":[179],"due":[180],"tendency":[183],"overrate":[185],"remaining":[188],"issues":[189],"capture":[191],"content":[193],"quality,":[194],"require":[197],"further":[198],"refinement.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":22}],"updated_date":"2026-04-07T14:57:38.498316","created_date":"2025-10-10T00:00:00"}