{"id":"https://openalex.org/W4213421742","doi":"https://doi.org/10.1145/3513140","title":"Automated Assessment in Computer Science Education: A State-of-the-Art Review","display_name":"Automated Assessment in Computer Science Education: A State-of-the-Art Review","publication_year":2022,"publication_date":"2022-02-24","ids":{"openalex":"https://openalex.org/W4213421742","doi":"https://doi.org/10.1145/3513140"},"language":"en","primary_location":{"id":"doi:10.1145/3513140","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3513140","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076620327","display_name":"Jos\u00e9 Carlos Paiva","orcid":"https://orcid.org/0000-0003-0394-0527"},"institutions":[{"id":"https://openalex.org/I4210166615","display_name":"INESC TEC","ror":"https://ror.org/05fa8ka61","country_code":"PT","type":"nonprofit","lineage":["https://openalex.org/I4210125590","https://openalex.org/I4210166615"]}],"countries":["PT"],"is_corresponding":true,"raw_author_name":"Jos\u00e9 Carlos Paiva","raw_affiliation_strings":["CRACS-INESC TEC and DCC-FCUP, Porto, Portugal"],"raw_orcid":"https://orcid.org/0000-0003-0394-0527","affiliations":[{"raw_affiliation_string":"CRACS-INESC TEC and DCC-FCUP, Porto, Portugal","institution_ids":["https://openalex.org/I4210166615"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038312045","display_name":"Jos\u00e9 Paulo Leal","orcid":"https://orcid.org/0000-0002-8409-0300"},"institutions":[{"id":"https://openalex.org/I4210166615","display_name":"INESC TEC","ror":"https://ror.org/05fa8ka61","country_code":"PT","type":"nonprofit","lineage":["https://openalex.org/I4210125590","https://openalex.org/I4210166615"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Jos\u00e9 Paulo Leal","raw_affiliation_strings":["CRACS-INESC TEC and DCC-FCUP, Porto, Portugal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CRACS-INESC TEC and DCC-FCUP, Porto, Portugal","institution_ids":["https://openalex.org/I4210166615"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029477501","display_name":"\u00c1lvaro Figueira","orcid":"https://orcid.org/0000-0002-0507-7504"},"institutions":[{"id":"https://openalex.org/I4210166615","display_name":"INESC TEC","ror":"https://ror.org/05fa8ka61","country_code":"PT","type":"nonprofit","lineage":["https://openalex.org/I4210125590","https://openalex.org/I4210166615"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"\u00c1lvaro Figueira","raw_affiliation_strings":["CRACS-INESC TEC and DCC-FCUP, Porto, Portugal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CRACS-INESC TEC and DCC-FCUP, Porto, Portugal","institution_ids":["https://openalex.org/I4210166615"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5076620327"],"corresponding_institution_ids":["https://openalex.org/I4210166615"],"apc_list":null,"apc_paid":null,"fwci":47.9002,"has_fulltext":false,"cited_by_count":235,"citation_normalized_percentile":{"value":0.99835554,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"22","issue":"3","first_page":"1","last_page":"40"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9879000186920166,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.8026976585388184},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7242482900619507},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7079864740371704},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4160218834877014},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.3515778183937073},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3469200134277344},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.18823930621147156},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1498822271823883}],"concepts":[{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.8026976585388184},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7242482900619507},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7079864740371704},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4160218834877014},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3515778183937073},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3469200134277344},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.18823930621147156},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1498822271823883}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3513140","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3513140","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7599999904632568,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W1561264025","https://openalex.org/W2253400989","https://openalex.org/W2380113066","https://openalex.org/W2461395154","https://openalex.org/W2575196328","https://openalex.org/W2603445299","https://openalex.org/W2721581421","https://openalex.org/W3048105396","https://openalex.org/W3097813857","https://openalex.org/W4238444100","https://openalex.org/W4246806392"],"related_works":["https://openalex.org/W1964661231","https://openalex.org/W3110264473","https://openalex.org/W4254960163","https://openalex.org/W2032810564","https://openalex.org/W2370831213","https://openalex.org/W2972807713","https://openalex.org/W4315704968","https://openalex.org/W2100255360","https://openalex.org/W2107506952","https://openalex.org/W3135287016"],"abstract_inverted_index":{"Practical":[0],"programming":[1,41],"competencies":[2],"are":[3,150],"critical":[4],"to":[5,52,92,168,176,223],"the":[6,19,61,68,87,94,122,129,132,164,187,190,193,201,217,221,252,256,260],"success":[7],"in":[8,73,78,128,192],"computer":[9],"science":[10],"(CS)":[11],"education":[12,107],"and":[13,31,33,82,126,139,172,216,225,239,262],"go-to-market":[14],"of":[15,22,28,40,70,90,96,124,159,189,196,204,213,231,259],"fresh":[16],"graduates.":[17],"Acquiring":[18],"required":[20],"level":[21],"skills":[23],"is":[24,49,112],"a":[25,37,74,79,102,110],"long":[26,100],"journey":[27],"discovery,":[29],"trial":[30],"error,":[32],"optimization":[34],"seeking":[35],"through":[36],"broad":[38],"range":[39],"activities":[42],"that":[43,54,60],"learners":[44],"must":[45],"perform":[46],"themselves.":[47],"It":[48],"not":[50],"reasonable":[51],"consider":[53],"teachers":[55],"could":[56],"evaluate":[57],"all":[58],"attempts":[59],"average":[62],"learner":[63],"should":[64],"develop":[65],"multiplied":[66],"by":[67],"number":[69],"students":[71],"enrolled":[72],"course,":[75],"much":[76],"less":[77],"timely,":[80],"deep,":[81],"fair":[83],"fashion.":[84],"Unsurprisingly,":[85],"exploring":[86],"formal":[88],"structure":[89],"programs":[91],"automate":[93],"assessment":[95,195],"certain":[97],"features":[98],"has":[99,241],"been":[101,242],"hot":[103],"topic":[104],"among":[105,141],"CS":[106,197],"practitioners.":[108],"Assessing":[109],"program":[111],"considerably":[113],"more":[114],"complex":[115],"than":[116],"asserting":[117],"its":[118],"functional":[119],"correctness,":[120],"as":[121],"proliferation":[123],"tools":[125],"techniques":[127,210,238],"literature":[130],"over":[131],"past":[133],"decades":[134],"indicates.":[135],"Program":[136],"efficiency,":[137],"behavior,":[138],"readability,":[140],"many":[142],"other":[143,249],"features,":[144],"assessed":[145],"either":[146],"statically":[147],"or":[148],"dynamically,":[149],"now":[151],"also":[152],"relevant":[153],"for":[154],"automatic":[155],"evaluation.":[156],"The":[157],"outcome":[158],"an":[160],"evaluation":[161],"evolved":[162],"from":[163,251],"primordial":[165],"Boolean":[166],"values":[167],"information":[169,218],"about":[170],"errors":[171],"tips":[173],"on":[174,200,235],"how":[175],"advance,":[177],"possibly":[178],"taking":[179],"into":[180],"account":[181],"similar":[182],"solutions.":[183],"This":[184],"work":[185],"surveys":[186],"state":[188],"art":[191],"automated":[194,232],"assignments,":[198],"focusing":[199],"supported":[202],"types":[203],"exercises,":[205],"security":[206],"measures":[207],"adopted,":[208],"testing":[209],"used,":[211],"type":[212],"feedback":[214],"produced,":[215],"they":[219],"offer":[220],"teacher":[222],"understand":[224],"optimize":[226],"learning.":[227],"A":[228],"new":[229],"era":[230],"assessment,":[233],"capitalizing":[234],"static":[236],"analysis":[237],"containerization,":[240],"identified.":[243],"Furthermore,":[244],"this":[245],"review":[246],"presents":[247],"several":[248],"findings":[250],"conducted":[253],"review,":[254],"discusses":[255],"current":[257],"challenges":[258],"field,":[261],"proposes":[263],"some":[264],"future":[265],"research":[266],"directions.":[267]},"counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":76},{"year":2024,"cited_by_count":60},{"year":2023,"cited_by_count":66},{"year":2022,"cited_by_count":18}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}