{"id":"https://openalex.org/W3188564437","doi":"https://doi.org/10.1109/access.2021.3100890","title":"A Systematic Review of the Effects of Automatic Scoring and Automatic Feedback in Educational Settings","display_name":"A Systematic Review of the Effects of Automatic Scoring and Automatic Feedback in Educational Settings","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3188564437","doi":"https://doi.org/10.1109/access.2021.3100890","mag":"3188564437"},"language":"en","primary_location":{"id":"doi:10.1109/access.2021.3100890","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3100890","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09500124.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"review","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09500124.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054946605","display_name":"Marcelo Guerra Hahn","orcid":"https://orcid.org/0000-0003-4351-9390"},"institutions":[{"id":"https://openalex.org/I4210116615","display_name":"Universidad Internacional De La Rioja","ror":"https://ror.org/029gnnp81","country_code":"ES","type":"education","lineage":["https://openalex.org/I4210116615"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Marcelo Guerra Hahn","raw_affiliation_strings":["Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain"],"raw_orcid":"https://orcid.org/0000-0003-4351-9390","affiliations":[{"raw_affiliation_string":"Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain","institution_ids":["https://openalex.org/I4210116615"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072723542","display_name":"Silvia Baldiris","orcid":"https://orcid.org/0000-0002-6818-3427"},"institutions":[{"id":"https://openalex.org/I4210116615","display_name":"Universidad Internacional De La Rioja","ror":"https://ror.org/029gnnp81","country_code":"ES","type":"education","lineage":["https://openalex.org/I4210116615"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Silvia Margarita Baldiris Navarro","raw_affiliation_strings":["Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain"],"raw_orcid":"https://orcid.org/0000-0002-6818-3427","affiliations":[{"raw_affiliation_string":"Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain","institution_ids":["https://openalex.org/I4210116615"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020133360","display_name":"Luis de\u2010la\u2010Fuente\u2010Valent\u00edn","orcid":"https://orcid.org/0000-0001-9727-315X"},"institutions":[{"id":"https://openalex.org/I4210116615","display_name":"Universidad Internacional De La Rioja","ror":"https://ror.org/029gnnp81","country_code":"ES","type":"education","lineage":["https://openalex.org/I4210116615"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Luis De La Fuente Valentin","raw_affiliation_strings":["Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain"],"raw_orcid":"https://orcid.org/0000-0001-9727-315X","affiliations":[{"raw_affiliation_string":"Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain","institution_ids":["https://openalex.org/I4210116615"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047213931","display_name":"Daniel Burgos","orcid":"https://orcid.org/0000-0003-0498-1101"},"institutions":[{"id":"https://openalex.org/I4210116615","display_name":"Universidad Internacional De La Rioja","ror":"https://ror.org/029gnnp81","country_code":"ES","type":"education","lineage":["https://openalex.org/I4210116615"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Daniel Burgos","raw_affiliation_strings":["Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain"],"raw_orcid":"https://orcid.org/0000-0003-0498-1101","affiliations":[{"raw_affiliation_string":"Escuela Superior de Ingenier\u00eda y Tecnolog\u00eda (ESIT), Universidad Internacional de La Rioja (UNIR), Logroño, La Rioja, Spain","institution_ids":["https://openalex.org/I4210116615"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210116615"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":1.9729,"has_fulltext":true,"cited_by_count":100,"citation_normalized_percentile":{"value":0.84579439,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"9","issue":null,"first_page":"108190","last_page":"108198"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.9732000231742859,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9728000164031982,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.8269405364990234},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8170238733291626},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.4752245545387268},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4063408076763153},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.38892602920532227},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3417831063270569},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3282623291015625},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2443515658378601}],"concepts":[{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.8269405364990234},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8170238733291626},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.4752245545387268},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4063408076763153},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.38892602920532227},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3417831063270569},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3282623291015625},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2443515658378601},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2021.3100890","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3100890","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09500124.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:981a4ed188f64d26bdf4ddaf122fe8a1","is_oa":true,"landing_page_url":"https://doaj.org/article/981a4ed188f64d26bdf4ddaf122fe8a1","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 9, Pp 108190-108198 (2021)","raw_type":"article"},{"id":"pmh:oai:reunir.unir.net:123456789/12620","is_oa":true,"landing_page_url":"https://reunir.unir.net/handle/123456789/12620","pdf_url":null,"source":{"id":"https://openalex.org/S4306400329","display_name":"Re-Unir (International University of La Rioja)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2021.3100890","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3100890","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9312710/09500124.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.6700000166893005,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G5228876551","display_name":null,"funder_award_id":"B0036","funder_id":"https://openalex.org/F4320317360","funder_display_name":"Universidad Internacional de La Rioja"}],"funders":[{"id":"https://openalex.org/F4320317360","display_name":"Universidad Internacional de La Rioja","ror":"https://ror.org/029gnnp81"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3188564437.pdf","grobid_xml":"https://content.openalex.org/works/W3188564437.grobid-xml"},"referenced_works_count":130,"referenced_works":["https://openalex.org/W165547694","https://openalex.org/W1940128419","https://openalex.org/W1986183039","https://openalex.org/W2089010774","https://openalex.org/W2118328848","https://openalex.org/W2293069149","https://openalex.org/W2400080339","https://openalex.org/W2408482380","https://openalex.org/W2428797009","https://openalex.org/W2515992743","https://openalex.org/W2523554497","https://openalex.org/W2531834029","https://openalex.org/W2547115911","https://openalex.org/W2562999332","https://openalex.org/W2564237127","https://openalex.org/W2567547739","https://openalex.org/W2568837515","https://openalex.org/W2572899893","https://openalex.org/W2587286194","https://openalex.org/W2592374163","https://openalex.org/W2604365061","https://openalex.org/W2607413545","https://openalex.org/W2622208228","https://openalex.org/W2623867476","https://openalex.org/W2624412722","https://openalex.org/W2724262105","https://openalex.org/W2729177083","https://openalex.org/W2739854726","https://openalex.org/W2744421942","https://openalex.org/W2747186544","https://openalex.org/W2763958978","https://openalex.org/W2767294613","https://openalex.org/W2773715656","https://openalex.org/W2783147400","https://openalex.org/W2793675870","https://openalex.org/W2795122340","https://openalex.org/W2798812031","https://openalex.org/W2799100597","https://openalex.org/W2799145797","https://openalex.org/W2804462692","https://openalex.org/W2808827600","https://openalex.org/W2808830614","https://openalex.org/W2811142795","https://openalex.org/W2883694211","https://openalex.org/W2885635790","https://openalex.org/W2887559623","https://openalex.org/W2888984439","https://openalex.org/W2900327614","https://openalex.org/W2905133030","https://openalex.org/W2906948151","https://openalex.org/W2909437461","https://openalex.org/W2910369897","https://openalex.org/W2910871123","https://openalex.org/W2914394663","https://openalex.org/W2914823529","https://openalex.org/W2921053723","https://openalex.org/W2921395495","https://openalex.org/W2921466487","https://openalex.org/W2940135214","https://openalex.org/W2940725132","https://openalex.org/W2940799513","https://openalex.org/W2940967127","https://openalex.org/W2943016562","https://openalex.org/W2945555654","https://openalex.org/W2946155872","https://openalex.org/W2946262266","https://openalex.org/W2947843165","https://openalex.org/W2950619306","https://openalex.org/W2951263191","https://openalex.org/W2951686276","https://openalex.org/W2954390611","https://openalex.org/W2955620394","https://openalex.org/W2955915380","https://openalex.org/W2958619243","https://openalex.org/W2962747642","https://openalex.org/W2963210840","https://openalex.org/W2964446959","https://openalex.org/W2966699125","https://openalex.org/W2967730738","https://openalex.org/W2969515974","https://openalex.org/W2969767290","https://openalex.org/W2971128324","https://openalex.org/W2971937231","https://openalex.org/W2976776212","https://openalex.org/W2979183952","https://openalex.org/W2979581057","https://openalex.org/W2980414158","https://openalex.org/W2981863007","https://openalex.org/W2983995213","https://openalex.org/W2987183241","https://openalex.org/W2989613245","https://openalex.org/W2994675679","https://openalex.org/W2994844966","https://openalex.org/W2997309766","https://openalex.org/W2998476166","https://openalex.org/W2999942862","https://openalex.org/W3004064327","https://openalex.org/W3004217031","https://openalex.org/W3006446134","https://openalex.org/W3006766033","https://openalex.org/W3007452953","https://openalex.org/W3007803772","https://openalex.org/W3007982904","https://openalex.org/W3011785238","https://openalex.org/W3012071850","https://openalex.org/W3012075170","https://openalex.org/W3026791757","https://openalex.org/W3028958483","https://openalex.org/W3030494876","https://openalex.org/W3043122700","https://openalex.org/W3045557678","https://openalex.org/W3047890686","https://openalex.org/W3048614748","https://openalex.org/W3049385593","https://openalex.org/W3049578509","https://openalex.org/W3082971966","https://openalex.org/W3087431152","https://openalex.org/W3102325394","https://openalex.org/W3103878770","https://openalex.org/W3110104823","https://openalex.org/W3120468046","https://openalex.org/W4321847712","https://openalex.org/W6729220650","https://openalex.org/W6750535278","https://openalex.org/W6750936052","https://openalex.org/W6762182156","https://openalex.org/W6763264074","https://openalex.org/W6767909737","https://openalex.org/W6786853809","https://openalex.org/W6849780589"],"related_works":["https://openalex.org/W2064165679","https://openalex.org/W1588461101","https://openalex.org/W2885058781","https://openalex.org/W3208525924","https://openalex.org/W1552490082","https://openalex.org/W2189583758","https://openalex.org/W3009904625","https://openalex.org/W1983121850","https://openalex.org/W3088126669","https://openalex.org/W1516538226"],"abstract_inverted_index":{"Automatic":[0],"scoring":[1,88,116],"and":[2,64,77,81,89,104,117,151,153],"feedback":[3,90,118],"tools":[4,14,256],"have":[5,119,164,186],"become":[6],"critical":[7],"components":[8],"of":[9,43,51,56,71,86,100,131,138,206,215,240],"online":[10,32],"learning":[11,24,93,207,261],"proliferation.":[12],"These":[13],"range":[15],"from":[16,53],"multiple-choice":[17],"questions":[18,57,106],"to":[19,58,174,201,225,233,246,250,253,257,263],"grading":[20,61,150],"essays":[21],"using":[22],"machine":[23],"(ML).":[25],"Learning":[26],"environments":[27],"such":[28,219],"as":[29,91],"massive":[30],"open":[31,105],"courses":[33],"(MOOCs)":[34],"would":[35],"not":[36,180,187],"be":[37,223],"possible":[38],"without":[39,133],"them.":[40],"The":[41,111,122,167],"usage":[42],"this":[44,108],"mechanism":[45],"has":[46],"brought":[47],"many":[48,120],"exciting":[49],"areas":[50],"study,":[52],"the":[54,59,69,84,101,129,141,146,159,182,192,199,203,208,213,226,236],"design":[55],"ML":[60],"tools\u2019":[62],"precision":[63],"accuracy.":[65],"This":[66,95],"paper":[67],"analyzes":[68],"findings":[70],"125":[72],"studies":[73],"published":[74],"in":[75,107,156],"journals":[76],"proceedings":[78],"between":[79,148],"2016":[80],"2020":[82],"on":[83],"usages":[85],"automatic":[87,115],"a":[92,135,172,216,259],"tool.":[94],"analysis":[96],"gives":[97],"an":[98,220,244],"overview":[99],"trends,":[102],"challenges,":[103],"research":[109],"area.":[110],"results":[112],"indicate":[113],"that":[114,178],"advantages.":[121],"most":[123],"important":[124],"benefits":[125],"include":[126],"enabling":[127],"scaling":[128],"number":[130,137],"students":[132,232],"adding":[134],"proportional":[136],"instructors,":[139],"improving":[140],"student":[142,200],"experience":[143,262],"by":[144],"reducing":[145],"time":[147],"submission":[149],"feedback,":[152],"removing":[154],"bias":[155],"scoring.":[157],"On":[158],"other":[160],"hand,":[161],"these":[162,241,255],"technologies":[163,252],"some":[165],"drawbacks.":[166],"main":[168],"problem":[169],"is":[170,196],"creating":[171],"disincentive":[173],"develop":[175],"innovative":[176],"answers":[177],"do":[179],"match":[181],"expected":[183],"one":[184],"or":[185],"been":[188],"considered":[189],"when":[190],"preparing":[191],"problem.":[193,237],"Another":[194],"drawback":[195],"potentially":[197],"training":[198],"answer":[202,221],"question":[204],"instead":[205],"concepts.":[209],"With":[210],"this,":[211],"given":[212],"existence":[214],"correct":[217],"answer,":[218],"could":[222],"leaked":[224],"internet,":[227],"making":[228],"it":[229],"easier":[230],"for":[231],"avoid":[234],"solving":[235],"Overall,":[238],"each":[239],"drawbacks":[242],"presents":[243],"opportunity":[245],"look":[247],"at":[248],"ways":[249],"improve":[251],"use":[254],"provide":[258],"better":[260],"students.":[264]},"counts_by_year":[{"year":2026,"cited_by_count":22},{"year":2025,"cited_by_count":33},{"year":2024,"cited_by_count":25},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":4}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2021-08-16T00:00:00"}