{"id":"https://openalex.org/W4377095850","doi":"https://doi.org/10.1007/s10115-023-01892-9","title":"GradeAid: a framework for automatic short answers grading in educational contexts\u2014design, implementation and evaluation","display_name":"GradeAid: a framework for automatic short answers grading in educational contexts\u2014design, implementation and evaluation","publication_year":2023,"publication_date":"2023-05-19","ids":{"openalex":"https://openalex.org/W4377095850","doi":"https://doi.org/10.1007/s10115-023-01892-9","pmid":"https://pubmed.ncbi.nlm.nih.gov/37361374"},"language":"en","primary_location":{"id":"doi:10.1007/s10115-023-01892-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10115-023-01892-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10115-023-01892-9.pdf","source":{"id":"https://openalex.org/S81770430","display_name":"Knowledge and Information Systems","issn_l":"0219-1377","issn":["0219-1377","0219-3116"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Knowledge and Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10115-023-01892-9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001837293","display_name":"Emiliano del Gobbo","orcid":"https://orcid.org/0000-0003-1088-7306"},"institutions":[{"id":"https://openalex.org/I106296451","display_name":"University of Foggia","ror":"https://ror.org/01xtv3204","country_code":"IT","type":"education","lineage":["https://openalex.org/I106296451"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Emiliano del\u00a0Gobbo","raw_affiliation_strings":["Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121 Foggia, FG Italy","Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121, Foggia, FG, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121 Foggia, FG Italy","institution_ids":["https://openalex.org/I106296451"]},{"raw_affiliation_string":"Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121, Foggia, FG, Italy","institution_ids":["https://openalex.org/I106296451"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085259062","display_name":"Alfonso Guarino","orcid":"https://orcid.org/0000-0002-9055-9689"},"institutions":[{"id":"https://openalex.org/I106296451","display_name":"University of Foggia","ror":"https://ror.org/01xtv3204","country_code":"IT","type":"education","lineage":["https://openalex.org/I106296451"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Alfonso Guarino","raw_affiliation_strings":["Department of Humanities, University of Foggia, Via Arpi, 176, 71121 Foggia, FG Italy","Department of Humanities, University of Foggia, Via Arpi, 176, 71121, Foggia, FG, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Humanities, University of Foggia, Via Arpi, 176, 71121 Foggia, FG Italy","institution_ids":["https://openalex.org/I106296451"]},{"raw_affiliation_string":"Department of Humanities, University of Foggia, Via Arpi, 176, 71121, Foggia, FG, Italy","institution_ids":["https://openalex.org/I106296451"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040736855","display_name":"Barbara Cafarelli","orcid":"https://orcid.org/0000-0002-7385-4213"},"institutions":[{"id":"https://openalex.org/I106296451","display_name":"University of Foggia","ror":"https://ror.org/01xtv3204","country_code":"IT","type":"education","lineage":["https://openalex.org/I106296451"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Barbara Cafarelli","raw_affiliation_strings":["Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121 Foggia, FG Italy","Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121, Foggia, FG, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121 Foggia, FG Italy","institution_ids":["https://openalex.org/I106296451"]},{"raw_affiliation_string":"Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121, Foggia, FG, Italy","institution_ids":["https://openalex.org/I106296451"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064815039","display_name":"Luca Grilli","orcid":"https://orcid.org/0000-0003-0931-2054"},"institutions":[{"id":"https://openalex.org/I106296451","display_name":"University of Foggia","ror":"https://ror.org/01xtv3204","country_code":"IT","type":"education","lineage":["https://openalex.org/I106296451"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Luca Grilli","raw_affiliation_strings":["Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121 Foggia, FG Italy","Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121, Foggia, FG, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121 Foggia, FG Italy","institution_ids":["https://openalex.org/I106296451"]},{"raw_affiliation_string":"Department of Economics, Management and Territory, University of Foggia, Via da Zara, 11, 71121, Foggia, FG, Italy","institution_ids":["https://openalex.org/I106296451"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5085259062"],"corresponding_institution_ids":["https://openalex.org/I106296451"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":4.3715,"has_fulltext":true,"cited_by_count":33,"citation_normalized_percentile":{"value":0.95534681,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"65","issue":"10","first_page":"4295","last_page":"4334"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.7864216566085815},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7674524784088135},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6125183701515198},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.5169350504875183},{"id":"https://openalex.org/keywords/learning-analytics","display_name":"Learning analytics","score":0.4818214476108551},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.46316325664520264},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4465984106063843},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3477070927619934},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.3316122889518738},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3243842124938965}],"concepts":[{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.7864216566085815},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7674524784088135},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6125183701515198},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.5169350504875183},{"id":"https://openalex.org/C2777648619","wikidata":"https://www.wikidata.org/wiki/Q2845208","display_name":"Learning analytics","level":2,"score":0.4818214476108551},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.46316325664520264},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4465984106063843},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3477070927619934},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3316122889518738},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3243842124938965},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1007/s10115-023-01892-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10115-023-01892-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10115-023-01892-9.pdf","source":{"id":"https://openalex.org/S81770430","display_name":"Knowledge and Information Systems","issn_l":"0219-1377","issn":["0219-1377","0219-3116"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Knowledge and Information Systems","raw_type":"journal-article"},{"id":"pmid:37361374","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37361374","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Knowledge and information systems","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:10197042","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10197042","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10197042/pdf/10115_2023_Article_1892.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Knowl Inf Syst","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1007/s10115-023-01892-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10115-023-01892-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10115-023-01892-9.pdf","source":{"id":"https://openalex.org/S81770430","display_name":"Knowledge and Information Systems","issn_l":"0219-1377","issn":["0219-1377","0219-3116"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Knowledge and Information Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8799999952316284}],"awards":[],"funders":[{"id":"https://openalex.org/F4320316353","display_name":"Universit\u00e0 degli Studi di Foggia","ror":"https://ror.org/01xtv3204"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4377095850.pdf"},"referenced_works_count":68,"referenced_works":["https://openalex.org/W55204438","https://openalex.org/W1577581905","https://openalex.org/W1967082761","https://openalex.org/W2018066946","https://openalex.org/W2025595753","https://openalex.org/W2141373701","https://openalex.org/W2470673105","https://openalex.org/W2508865106","https://openalex.org/W2569936787","https://openalex.org/W2739522917","https://openalex.org/W2793303358","https://openalex.org/W2883991406","https://openalex.org/W2900237823","https://openalex.org/W2900327614","https://openalex.org/W2914823529","https://openalex.org/W2923421605","https://openalex.org/W2936061554","https://openalex.org/W2936258623","https://openalex.org/W2938619571","https://openalex.org/W2947181839","https://openalex.org/W2951166026","https://openalex.org/W2955427418","https://openalex.org/W2955877667","https://openalex.org/W2962851685","https://openalex.org/W2964125683","https://openalex.org/W2966193711","https://openalex.org/W2970641574","https://openalex.org/W2970812170","https://openalex.org/W2973748901","https://openalex.org/W2975540156","https://openalex.org/W2998900056","https://openalex.org/W2999896861","https://openalex.org/W3005804827","https://openalex.org/W3010884778","https://openalex.org/W3017029353","https://openalex.org/W3022395572","https://openalex.org/W3023064165","https://openalex.org/W3024396728","https://openalex.org/W3028435993","https://openalex.org/W3030494876","https://openalex.org/W3033475071","https://openalex.org/W3037461693","https://openalex.org/W3037797173","https://openalex.org/W3040237235","https://openalex.org/W3044788535","https://openalex.org/W3047969382","https://openalex.org/W3093526511","https://openalex.org/W3094233077","https://openalex.org/W3095483082","https://openalex.org/W3111899316","https://openalex.org/W3115331384","https://openalex.org/W3125048963","https://openalex.org/W3133894035","https://openalex.org/W3134300882","https://openalex.org/W3157187185","https://openalex.org/W3171975879","https://openalex.org/W3177035927","https://openalex.org/W3183296033","https://openalex.org/W3198520229","https://openalex.org/W3214073331","https://openalex.org/W4229977739","https://openalex.org/W4245055982","https://openalex.org/W4285247275","https://openalex.org/W4294214983","https://openalex.org/W4294768490","https://openalex.org/W4308987120","https://openalex.org/W4361026235","https://openalex.org/W6702248584"],"related_works":["https://openalex.org/W1987827786","https://openalex.org/W2799586942","https://openalex.org/W2504091800","https://openalex.org/W2331775400","https://openalex.org/W2816728186","https://openalex.org/W2804624249","https://openalex.org/W2560130217","https://openalex.org/W605203981","https://openalex.org/W4225313275","https://openalex.org/W3097749327"],"abstract_inverted_index":{"Abstract":[0],"Automatic":[1],"short":[2],"answer":[3],"grading":[4,47,62],"(ASAG),":[5],"a":[6,14,85,96,115,154,173,219],"hot":[7],"field":[8],"of":[9,38,46,87,98,126,131],"natural":[10],"language":[11],"understanding,":[12],"is":[13,120],"research":[15],"area":[16],"within":[17],"learning":[18],"analytics.":[19],"ASAG":[20,73,88],"solutions":[21,89],"are":[22,40,56,95],"conceived":[23],"to":[24,49,184,194],"offload":[25],"teachers":[26],"and":[27,43,63,128,157,160,171],"instructors,":[28],"especially":[29],"those":[30],"in":[31,100,106,188,225],"higher":[32],"education,":[33],"where":[34],"classes":[35],"with":[36,67,147],"hundreds":[37],"students":[39,66],"the":[41,44,60,83,101,123,132,185,189,198,226],"norm":[42],"task":[45],"(short)answers":[48],"open-ended":[50],"questionnaires":[51],"becomes":[52],"tougher.":[53],"Their":[54],"outcomes":[55],"precious":[57],"both":[58],"for":[59,64,117,178,222],"very":[61],"providing":[65],"\u201c":[68],"ad":[69],"hoc":[70],"\u201d":[71],"feedback.":[72],"proposals":[74],"have":[75,90],"also":[76],"enabled":[77],"different":[78],"intelligent":[79],"tutoring":[80],"systems.":[81],"Over":[82],"years,":[84],"variety":[86],"been":[91,164],"proposed,":[92],"still":[93],"there":[94],"series":[97],"gaps":[99],"literature":[102,190],"that":[103],"we":[104],"fill":[105],"this":[107],"paper.":[108],"The":[109],"present":[110],"work":[111],"proposes":[112],"GradeAid":[113,180],",":[114],"framework":[116],"ASAG.":[118],"It":[119],"based":[121,196],"on":[122,166,172,197],"joint":[124],"analysis":[125],"lexical":[127],"semantic":[129],"features":[130],"students\u2019":[133],"answers":[134],"through":[135],"state-of-the-art":[136],"regressors;":[137],"differently":[138],"from":[139],"any":[140],"other":[141],"previous":[142],"work,":[143],"(i)":[144],"it":[145,151,162,217],"copes":[146],"non-English":[148],"datasets,":[149],"(ii)":[150],"has":[152,163],"undergone":[153],"robust":[155],"validation":[156],"benchmarking":[158],"phase,":[159],"(iii)":[161],"tested":[165],"every":[167],"dataset":[168,175],"publicly":[169],"available":[170,177],"new":[174],"(now":[176],"researchers).":[179],"obtains":[181],"performance":[182],"comparable":[183],"systems":[186],"presented":[187],"(root-mean-squared":[191],"errors":[192],"down":[193],"0.25":[195],"specific":[199],"tuple":[200],"$$\\langle":[201],"$$":[202,209],"<mml:math":[203,210],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\">":[204,211],"<mml:mo>\u27e8</mml:mo>":[205],"</mml:math>":[206,213],"dataset-question":[207],"$$\\rangle":[208],"<mml:mo>\u27e9</mml:mo>":[212],").":[214],"We":[215],"argue":[216],"represents":[218],"strong":[220],"baseline":[221],"further":[223],"developments":[224],"field.":[227]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}