{"id":"https://openalex.org/W4407842016","doi":"https://doi.org/10.1145/3706468.3706527","title":"Can AI grade your essays? A comparative analysis of large language models and teacher ratings in multidimensional essay scoring","display_name":"Can AI grade your essays? A comparative analysis of large language models and teacher ratings in multidimensional essay scoring","alternate_titles":[],"publication_year":2025,"publication_date":"2025-03-03","ids":{"openalex":"https://openalex.org/W4407842016","doi":"https://doi.org/10.1145/3706468.3706527"},"language":"en","primary_location":{"id":"doi:10.1145/3706468.3706527","doi":"https://doi.org/10.1145/3706468.3706527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706527","pdf_url":null,"source":{"id":"https://openalex.org/S7407086742","display_name":"International Conference on Learning Analytics and Knowledge (LAK)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"parent_works":[{"id":"https://openalex.org/W4407842477","display_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","relationship":"volume","link_type":"volume"}],"lineage":["https://openalex.org/W4407842016","https://openalex.org/W4407842477"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3706468.3706527","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013346660","display_name":"Kathrin Se\u00dfler","orcid":"https://orcid.org/0000-0002-3380-4641","observed_orcids":["https://orcid.org/0000-0002-3380-4641"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Kathrin Se\u00dfler","raw_affiliation_strings":["Technical University of Munich, Munich, Germany"],"raw_orcid":"https://orcid.org/0000-0002-3380-4641","affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114987805","display_name":"Maurice F\u00fcrstenberg","orcid":"https://orcid.org/0009-0001-1090-9299","observed_orcids":["https://orcid.org/0009-0001-1090-9299"]},"institutions":[{"id":"https://openalex.org/I8087733","display_name":"University of T\u00fcbingen","ror":"https://ror.org/03a1kwz48","country_code":"DE","type":"education","lineage":["https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Maurice F\u00fcrstenberg","raw_affiliation_strings":["University of T\u00fcbingen, T\u00fcbingen, Germany"],"raw_orcid":"https://orcid.org/0009-0001-1090-9299","affiliations":[{"raw_affiliation_string":"University of T\u00fcbingen, T\u00fcbingen, Germany","institution_ids":["https://openalex.org/I8087733"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057566592","display_name":"Babette B\u00fchler","orcid":"https://orcid.org/0000-0003-1679-4979","observed_orcids":["https://orcid.org/0000-0003-1679-4979"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Babette B\u00fchler","raw_affiliation_strings":["Technical University of Munich, Munich, Germany"],"raw_orcid":"https://orcid.org/0000-0003-1679-4979","affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008809634","display_name":"Enkelejda Kasneci","orcid":"https://orcid.org/0000-0003-3146-4484","observed_orcids":["https://orcid.org/0000-0003-3146-4484"]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Enkelejda Kasneci","raw_affiliation_strings":["Technical University of Munich, Munich, Germany"],"raw_orcid":"https://orcid.org/0000-0003-3146-4484","affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":271.2298,"has_fulltext":true,"cited_by_count":132,"reviewed_by_count":0,"citation_normalized_percentile":{"value":0.99996235,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"462","last_page":"472"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.7470999956130981,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.7470999956130981,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.07280000299215317,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.042100001126527786,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-essay-scoring","display_name":"automated essay scoring","score":0.8140000104904175},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9959999918937683},{"id":"https://openalex.org/keywords/teacher-ratings","display_name":"teacher ratings","score":0.6389999985694885},{"id":"https://openalex.org/keywords/student-writing-assessment","display_name":"student writing assessment","score":0.6729999780654907},{"id":"https://openalex.org/keywords/internal-consistency","display_name":"internal consistency","score":0.5950000286102295},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.7409999966621399},{"id":"https://openalex.org/keywords/llama-3","display_name":"Llama 3","score":0.8130000233650208},{"id":"https://openalex.org/keywords/mixtral","display_name":"Mixtral","score":0.8429999947547913},{"id":"https://openalex.org/keywords/teacher-workload-reduction","display_name":"teacher workload reduction","score":0.5249999761581421},{"id":"https://openalex.org/keywords/content-quality","display_name":"content quality","score":0.3100000023841858}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6165816783905029},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5427610278129578},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4686267673969269},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3747377395629883},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.29604315757751465}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3706468.3706527","doi":"https://doi.org/10.1145/3706468.3706527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706527","pdf_url":null,"source":{"id":"https://openalex.org/S7407086742","display_name":"International Conference on Learning Analytics and Knowledge (LAK)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2411.16337","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.16337","pdf_url":"https://arxiv.org/pdf/2411.16337","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:epub.ub.uni-muenchen.de:126938","doi":null,"is_oa":true,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bvb:19-epub-126938-0","pdf_url":"https://epub.ub.uni-muenchen.de/126938/1/Paper_in_Proceedings.pdf","source":{"id":"https://openalex.org/S4306401845","display_name":"Open access LMU (Ludwid Maxmilian's Universitat Munchen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I8204097","host_organization_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","host_organization_lineage":["https://openalex.org/I8204097"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"doc-type:conferenceObject"},{"id":"doi:10.48550/arxiv.2411.16337","doi":"https://doi.org/10.48550/arxiv.2411.16337","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.16337","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3706468.3706527","doi":"https://doi.org/10.1145/3706468.3706527","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706527","pdf_url":null,"source":{"id":"https://openalex.org/S7407086742","display_name":"International Conference on Learning Analytics and Knowledge (LAK)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.9235000014305115,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407842016.pdf","grobid_xml":"https://content.openalex.org/works/W4407842016.grobid-xml"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W2055011070","https://openalex.org/W2106695994","https://openalex.org/W2141960018","https://openalex.org/W2327037637","https://openalex.org/W2506826885","https://openalex.org/W2526543293","https://openalex.org/W2952164904","https://openalex.org/W2963341956","https://openalex.org/W2964413085","https://openalex.org/W3114208363","https://openalex.org/W3144434203","https://openalex.org/W3170037077","https://openalex.org/W3196589716","https://openalex.org/W3201077663","https://openalex.org/W4226379858","https://openalex.org/W4229456833","https://openalex.org/W4285778680","https://openalex.org/W4310123629","https://openalex.org/W4310436400","https://openalex.org/W4323655724","https://openalex.org/W4327810158","https://openalex.org/W4366403195","https://openalex.org/W4380320806","https://openalex.org/W4382338912","https://openalex.org/W4384918448","https://openalex.org/W4385571232","https://openalex.org/W4385572239","https://openalex.org/W4386261929","https://openalex.org/W4386263510","https://openalex.org/W4387007101","https://openalex.org/W4387561528","https://openalex.org/W4387824566","https://openalex.org/W4389344855","https://openalex.org/W4389429742","https://openalex.org/W4391026420","https://openalex.org/W4392202731","https://openalex.org/W4393011212","https://openalex.org/W4395483820","https://openalex.org/W4399829620","https://openalex.org/W4402819599","https://openalex.org/W4405766390","https://openalex.org/W4406232026","https://openalex.org/W6908570072","https://openalex.org/W7128982879","https://openalex.org/W7133211372","https://openalex.org/W7133224126"],"related_works":["https://openalex.org/W4396806295","https://openalex.org/W7134280132","https://openalex.org/W4417104347","https://openalex.org/W7130585358","https://openalex.org/W7130426176","https://openalex.org/W7223051077","https://openalex.org/W7134165302","https://openalex.org/W6921774234","https://openalex.org/W7140687447","https://openalex.org/W4399334961"],"abstract_inverted_index":{"The":[0,125],"manual":[1],"assessment":[2,156],"and":[3,42,47,80,93,115,145],"grading":[4],"of":[5,44,60,73,149,170,195],"student":[6,53],"writing":[7],"is":[8],"a":[9,159],"time-consuming":[10],"yet":[11],"critical":[12],"task":[13],"for":[14,33,183],"teachers.":[15,34],"Recent":[16],"developments":[17],"in":[18,50,111,122,141],"generative":[19],"AI,":[20],"such":[21],"as":[22],"large":[23],"language":[24],"models,":[25],"offer":[26],"potential":[27],"solutions":[28],"to":[29,58,97,162,175,180,191],"facilitate":[30],"essay-scoring":[31],"tasks":[32],"In":[35],"our":[36],"study,":[37],"we":[38],"evaluate":[39],"the":[40,142,168,186],"performance":[41],"reliability":[43],"both":[45,112],"open-source":[46,109],"closed-source":[48],"LLMs":[49],"assessing":[51],"German":[52],"essays,":[54,171],"comparing":[55],"their":[56,181],"evaluations":[57],"those":[59],"37":[61],"teachers":[62],"across":[63],"10":[64],"pre-defined":[65],"criteria":[66],"(i.e.,":[67],"plot":[68],"logic,":[69],"expression).":[70],"A":[71],"corpus":[72],"20":[74],"real-world":[75],"essays":[76],"from":[77],"Year":[78],"7":[79],"8":[81],"students":[82],"was":[83],"analyzed":[84],"using":[85],"five":[86],"LLMs:":[87],"GPT-3.5,":[88],"GPT-4,":[89],"o1,":[90],"LLaMA":[91],"3-70B,":[92],"Mixtral":[94],"8x7B,":[95],"aiming":[96],"provide":[98],"in-depth":[99],"insights":[100],"into":[101],"LLMs'":[102],"scoring":[103],"capabilities.":[104],"Closed-source":[105],"GPT":[106],"models":[107,110,187],"outperform":[108],"internal":[113,147],"consistency":[114,148],"alignment":[116],"with":[117,138,173],"human":[118,139],"ratings,":[119],"particularly":[120],"excelling":[121],"language-related":[123,176],"criteria.":[124,177],"novel":[126],"o1":[127],"model":[128],"outperforms":[129],"all":[130],"other":[131],"LLMs,":[132],"achieving":[133],"Spearman's":[134],"$r":[135],"=":[136],".74$":[137],"assessments":[140],"overall":[143],"score,":[144],"an":[146],"$ICC=.80$.":[150],"These":[151],"findings":[152],"indicate":[153],"that":[154],"LLM-based":[155],"can":[157],"be":[158],"useful":[160],"tool":[161],"reduce":[163],"teacher":[164],"workload":[165],"by":[166],"supporting":[167],"evaluation":[169],"especially":[172],"regard":[174],"However,":[178],"due":[179],"tendency":[182],"higher":[184],"scores,":[185],"require":[188],"further":[189],"refinement":[190],"better":[192],"capture":[193],"aspects":[194],"content":[196],"quality.":[197]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4407842016","counts_by_year":[{"year":2026,"cited_by_count":88},{"year":2025,"cited_by_count":44}],"updated_date":"2026-11-22T12:46:00.618207","created_date":"2025-02-22T00:00:00"}
