{"id":"https://openalex.org/W4416900547","doi":"https://doi.org/10.1002/jcal.70160","title":"Automatic Short\u2010Answer Grading in Sustainability Education: <scp>AI</scp> \u2013Human Agreement","display_name":"Automatic Short\u2010Answer Grading in Sustainability Education: <scp>AI</scp> \u2013Human Agreement","publication_year":2025,"publication_date":"2025-12-02","ids":{"openalex":"https://openalex.org/W4416900547","doi":"https://doi.org/10.1002/jcal.70160"},"language":"en","primary_location":{"id":"doi:10.1002/jcal.70160","is_oa":false,"landing_page_url":"https://doi.org/10.1002/jcal.70160","pdf_url":null,"source":{"id":"https://openalex.org/S5743915","display_name":"Journal of Computer Assisted Learning","issn_l":"0266-4909","issn":["0266-4909","1365-2729"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Computer Assisted Learning","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003214688","display_name":"Emrah Emirtekin","orcid":"https://orcid.org/0000-0002-3970-4406"},"institutions":[{"id":"https://openalex.org/I41641357","display_name":"Ege University","ror":"https://ror.org/02eaafc18","country_code":"TR","type":"education","lineage":["https://openalex.org/I41641357"]}],"countries":["TR"],"is_corresponding":true,"raw_author_name":"Emrah Emirtekin","raw_affiliation_strings":["Center for Distance Education Application and Research, Ege University  \u0130zmir Turkey"],"raw_orcid":"https://orcid.org/0000-0002-3970-4406","affiliations":[{"raw_affiliation_string":"Center for Distance Education Application and Research, Ege University  \u0130zmir Turkey","institution_ids":["https://openalex.org/I41641357"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034659498","display_name":"Yasin \u00d6zarslan","orcid":"https://orcid.org/0000-0003-0831-6985"},"institutions":[{"id":"https://openalex.org/I59944920","display_name":"Ya\u015far University","ror":"https://ror.org/00dz1eb96","country_code":"TR","type":"education","lineage":["https://openalex.org/I59944920"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Yasin \u00d6zarslan","raw_affiliation_strings":["Department of Science Culture Yasar University  \u0130zmir Turkey"],"raw_orcid":"https://orcid.org/0000-0003-0831-6985","affiliations":[{"raw_affiliation_string":"Department of Science Culture Yasar University  \u0130zmir Turkey","institution_ids":["https://openalex.org/I59944920"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5003214688"],"corresponding_institution_ids":["https://openalex.org/I41641357"],"apc_list":{"value":3760,"currency":"USD","value_usd":3760},"apc_paid":null,"fwci":7.8776,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.97373479,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"42","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11835","display_name":"Sustainability in Higher Education","score":0.35420000553131104,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11835","display_name":"Sustainability in Higher Education","score":0.35420000553131104,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.0551999993622303,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12884","display_name":"Educational Assessment and Pedagogy","score":0.05090000107884407,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.6407999992370605},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.5958999991416931},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.550599992275238},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.5184000134468079},{"id":"https://openalex.org/keywords/intraclass-correlation","display_name":"Intraclass correlation","score":0.49320000410079956},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.484499990940094},{"id":"https://openalex.org/keywords/kappa","display_name":"Kappa","score":0.47859999537467957},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"Inter-rater reliability","score":0.4341999888420105},{"id":"https://openalex.org/keywords/cohens-kappa","display_name":"Cohen's kappa","score":0.430400013923645}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.6407999992370605},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.5958999991416931},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.550599992275238},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5401999950408936},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.5184000134468079},{"id":"https://openalex.org/C104709138","wikidata":"https://www.wikidata.org/wiki/Q1671540","display_name":"Intraclass correlation","level":3,"score":0.49320000410079956},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.484499990940094},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.47859999537467957},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.4341999888420105},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.430400013923645},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.39469999074935913},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3716999888420105},{"id":"https://openalex.org/C66204764","wikidata":"https://www.wikidata.org/wiki/Q219416","display_name":"Sustainability","level":2,"score":0.36629998683929443},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.3517000079154968},{"id":"https://openalex.org/C191147762","wikidata":"https://www.wikidata.org/wiki/Q186289","display_name":"Human reliability","level":3,"score":0.34470000863075256},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.34040001034736633},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.337799996137619},{"id":"https://openalex.org/C3017944768","wikidata":"https://www.wikidata.org/wiki/Q1450463","display_name":"Poison control","level":2,"score":0.3336000144481659},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.33230000734329224},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.33219999074935913},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3107999861240387},{"id":"https://openalex.org/C166735990","wikidata":"https://www.wikidata.org/wiki/Q1750812","display_name":"Human factors and ergonomics","level":3,"score":0.3003999888896942},{"id":"https://openalex.org/C61641136","wikidata":"https://www.wikidata.org/wiki/Q1107019","display_name":"Cognitive load","level":3,"score":0.2971999943256378},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C86658582","wikidata":"https://www.wikidata.org/wiki/Q1432778","display_name":"Social cognition","level":3,"score":0.2784000039100647},{"id":"https://openalex.org/C6057870","wikidata":"https://www.wikidata.org/wiki/Q5141248","display_name":"Cognitive test","level":3,"score":0.272599995136261},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.26980000734329224},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.26579999923706055},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.260699987411499},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.25609999895095825},{"id":"https://openalex.org/C169806903","wikidata":"https://www.wikidata.org/wiki/Q5937752","display_name":"Human error","level":2,"score":0.2554999887943268}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1002/jcal.70160","is_oa":false,"landing_page_url":"https://doi.org/10.1002/jcal.70160","pdf_url":null,"source":{"id":"https://openalex.org/S5743915","display_name":"Journal of Computer Assisted Learning","issn_l":"0266-4909","issn":["0266-4909","1365-2729"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Computer Assisted Learning","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W806933157","https://openalex.org/W2025828126","https://openalex.org/W2061056793","https://openalex.org/W2115389369","https://openalex.org/W2141058353","https://openalex.org/W2162821268","https://openalex.org/W2164777277","https://openalex.org/W2246543262","https://openalex.org/W2327037637","https://openalex.org/W2797953024","https://openalex.org/W2945282335","https://openalex.org/W2963341956","https://openalex.org/W3185341429","https://openalex.org/W4313410276","https://openalex.org/W4323655724","https://openalex.org/W4324046518","https://openalex.org/W4361204578","https://openalex.org/W4365388135","https://openalex.org/W4376059778","https://openalex.org/W4377242081","https://openalex.org/W4378193514","https://openalex.org/W4389437528","https://openalex.org/W4389733229","https://openalex.org/W4391145008","https://openalex.org/W4391689754","https://openalex.org/W4395069886","https://openalex.org/W4396806295","https://openalex.org/W4403260962","https://openalex.org/W4404271614","https://openalex.org/W4406216423","https://openalex.org/W4408374346","https://openalex.org/W4410202262","https://openalex.org/W4410550984","https://openalex.org/W4411072120","https://openalex.org/W6964970122"],"related_works":[],"abstract_inverted_index":{"ABSTRACT":[0],"Background":[1],"Sustainability":[2,68],"education":[3],"emphasises":[4],"critical":[5],"thinking":[6],"and":[7,50,123,133],"interdisciplinary":[8],"understanding,":[9],"making":[10],"the":[11,45],"assessment":[12],"of":[13,84],"students'":[14],"learning":[15],"outcomes":[16],"complex.":[17],"While":[18],"Large":[19],"Language":[20],"Models":[21],"(LLMs)":[22],"have":[23],"shown":[24],"promise":[25],"in":[26,31,61,136],"educational":[27],"assessment,":[28],"their":[29],"reliability":[30,147],"domains":[32],"requiring":[33],"contextual":[34],"reasoning\u2014such":[35],"as":[36,159],"sustainability\u2014remains":[37],"unclear.":[38],"Objectives":[39],"This":[40],"study":[41],"aims":[42],"to":[43,105,152],"evaluate":[44],"agreement":[46,75,128,157,166,174],"between":[47,131],"human":[48,101,134,196],"raters":[49,102,135],"several":[51],"LLMs":[52,132,188],"(GPT\u20104o,":[53],"Gemini":[54],"2.0":[55],"Flash,":[56],"DeepSeek":[57],"V3,":[58],"LLaMA":[59],"3.3)":[60],"assessing":[62],"short\u2010answer":[63,86],"responses":[64,87],"from":[65,100],"a":[66,91],"university\u2010level":[67],"course.":[69],"It":[70],"also":[71],"investigates":[72],"how":[73],"this":[74],"varies":[76],"across":[77],"cognitive":[78,160,184],"skill":[79],"levels.":[80],"Methods":[81],"A":[82],"total":[83,137,178],"232":[85],"were":[88,103],"evaluated":[89],"using":[90,108],"rubric":[92],"aligned":[93],"with":[94,163],"Bloom's":[95],"Revised":[96],"Taxonomy.":[97],"Consensus":[98],"scores":[99,107,138,179],"compared":[104],"LLM\u2010generated":[106],"multiple":[109],"statistical":[110],"measures,":[111],"including":[112],"Quadratic":[113],"Weighted":[114],"Kappa":[115],"(QWK),":[116],"Intraclass":[117],"Correlation":[118],"Coefficient":[119],"(ICC),":[120],"Pearson":[121],"correlation,":[122],"distributional":[124],"overlap.":[125],"Results":[126],"Moderate":[127],"was":[129,150,175],"found":[130],"(QWK:":[139],"0.585\u20130.640;":[140],"r":[141],":":[142,144],"0.660\u20130.668;":[143],"0.681\u20130.803).":[145],"Inter\u2010rater":[146],"among":[148],"humans":[149],"good":[151],"excellent":[153],"(ICC:":[154],"0.667\u20130.800).":[155],"Criterion\u2010level":[156],"declined":[158,181],"complexity":[161],"increased,":[162],"notably":[164],"low":[165],"on":[167,177],"evaluating":[168],"higher\u2010order":[169],"skills.":[170],"Conclusions":[171],"Overall,":[172],"LLM\u2013human":[173],"moderate":[176],"but":[180],"at":[182],"higher":[183],"levels,":[185],"indicating":[186],"that":[187],"are":[189],"suitable":[190],"for":[191,200],"basic":[192],"comprehension":[193],"checks":[194],"while":[195],"oversight":[197],"remains":[198],"necessary":[199],"complex":[201],"reasoning.":[202]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2025-12-03T00:07:38.036990","created_date":"2025-12-02T00:00:00"}