{"id":"https://openalex.org/W2151828850","doi":"https://doi.org/10.1145/1930464.1930480","title":"Review of recent systems for automatic assessment of programming assignments","display_name":"Review of recent systems for automatic assessment of programming assignments","publication_year":2010,"publication_date":"2010-10-28","ids":{"openalex":"https://openalex.org/W2151828850","doi":"https://doi.org/10.1145/1930464.1930480","mag":"2151828850"},"language":"en","primary_location":{"id":"doi:10.1145/1930464.1930480","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1930464.1930480","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 10th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033863786","display_name":"Petri Ihantola","orcid":"https://orcid.org/0000-0003-1197-7266"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Petri Ihantola","raw_affiliation_strings":["Aalto University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aalto University","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033932980","display_name":"Tuukka Ahoniemi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tuukka Ahoniemi","raw_affiliation_strings":["Digia Plc, Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Digia Plc, Finland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081776109","display_name":"Ville Karavirta","orcid":null},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Ville Karavirta","raw_affiliation_strings":["Aalto University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aalto University","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044303151","display_name":"Otto Sepp\u00e4l\u00e4","orcid":"https://orcid.org/0000-0003-4694-9580"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Otto Sepp\u00e4l\u00e4","raw_affiliation_strings":["Aalto University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aalto University","institution_ids":["https://openalex.org/I9927081"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":31.1145,"has_fulltext":false,"cited_by_count":545,"citation_normalized_percentile":{"value":0.99587567,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"86","last_page":"93"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9818000197410583,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.9793999791145325,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7584929466247559},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5923704504966736},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4900032579898834},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.4388091564178467},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4298652112483978},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.41036462783813477},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1683717966079712}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7584929466247559},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5923704504966736},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4900032579898834},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4388091564178467},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4298652112483978},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.41036462783813477},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1683717966079712},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1930464.1930480","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1930464.1930480","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 10th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.4000000059604645}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":92,"referenced_works":["https://openalex.org/W47328078","https://openalex.org/W114844246","https://openalex.org/W122259140","https://openalex.org/W156150494","https://openalex.org/W160252601","https://openalex.org/W167372973","https://openalex.org/W183770118","https://openalex.org/W347262446","https://openalex.org/W379711631","https://openalex.org/W1494744305","https://openalex.org/W1509202185","https://openalex.org/W1548906879","https://openalex.org/W1592877999","https://openalex.org/W1966463724","https://openalex.org/W1967897571","https://openalex.org/W1970459197","https://openalex.org/W1971621932","https://openalex.org/W1971973232","https://openalex.org/W1975350221","https://openalex.org/W1975881544","https://openalex.org/W1976546103","https://openalex.org/W1977668552","https://openalex.org/W1986245503","https://openalex.org/W1987116959","https://openalex.org/W1989065480","https://openalex.org/W1991140109","https://openalex.org/W1993494146","https://openalex.org/W2002799336","https://openalex.org/W2005134703","https://openalex.org/W2011607672","https://openalex.org/W2020126579","https://openalex.org/W2024866554","https://openalex.org/W2026432604","https://openalex.org/W2030175360","https://openalex.org/W2030888081","https://openalex.org/W2034129600","https://openalex.org/W2036586008","https://openalex.org/W2039670527","https://openalex.org/W2042764688","https://openalex.org/W2043503760","https://openalex.org/W2043542190","https://openalex.org/W2044529823","https://openalex.org/W2045071089","https://openalex.org/W2056580788","https://openalex.org/W2057150268","https://openalex.org/W2065315280","https://openalex.org/W2066033104","https://openalex.org/W2067391762","https://openalex.org/W2069249897","https://openalex.org/W2070405152","https://openalex.org/W2077539104","https://openalex.org/W2079496145","https://openalex.org/W2080795584","https://openalex.org/W2082112582","https://openalex.org/W2082434090","https://openalex.org/W2089010774","https://openalex.org/W2089172953","https://openalex.org/W2095423622","https://openalex.org/W2095637735","https://openalex.org/W2103802898","https://openalex.org/W2108365077","https://openalex.org/W2118328848","https://openalex.org/W2120411502","https://openalex.org/W2121191865","https://openalex.org/W2122312470","https://openalex.org/W2125656372","https://openalex.org/W2132874666","https://openalex.org/W2151724062","https://openalex.org/W2156327292","https://openalex.org/W2160081537","https://openalex.org/W2160950252","https://openalex.org/W2165005075","https://openalex.org/W2196854537","https://openalex.org/W2335861685","https://openalex.org/W2494680890","https://openalex.org/W2621728010","https://openalex.org/W2912941107","https://openalex.org/W2998537529","https://openalex.org/W2998939050","https://openalex.org/W2999919487","https://openalex.org/W3033820246","https://openalex.org/W3162685574","https://openalex.org/W4205582695","https://openalex.org/W4205785435","https://openalex.org/W4210377336","https://openalex.org/W4232970258","https://openalex.org/W4233466850","https://openalex.org/W4242003664","https://openalex.org/W4243488715","https://openalex.org/W4251800768","https://openalex.org/W4285719527","https://openalex.org/W6703328252"],"related_works":["https://openalex.org/W380379685","https://openalex.org/W2062838854","https://openalex.org/W1536049545","https://openalex.org/W574531438","https://openalex.org/W3123265518","https://openalex.org/W636238771","https://openalex.org/W1989140077","https://openalex.org/W3125325772","https://openalex.org/W286222301","https://openalex.org/W4367833263"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"a":[3,68,89],"systematic":[4],"literature":[5],"review":[6],"of":[7,12,43,46,70],"the":[8,21,25,29,37,40,52,86,103,107,116],"recent":[9],"(2006--2010)":[10],"development":[11],"automatic":[13],"assessment":[14],"tools":[15,26],"for":[16,51,106],"programming":[17],"exercises.":[18],"We":[19,64],"discuss":[20],"major":[22],"features":[23,48],"that":[24,77,93],"support":[27],"and":[28,39,61,119],"different":[30],"approaches":[31],"they":[32],"are":[33,49,78,98,131],"using":[34],"both":[35],"from":[36,128],"pedagogical":[38],"technical":[41],"point":[42],"view.":[44],"Examples":[45],"these":[47],"ways":[50],"teacher":[53],"to":[54,80],"define":[55],"tests,":[56],"resubmission":[57],"policies,":[58],"security":[59],"issues,":[60],"so":[62],"forth.":[63],"have":[65],"also":[66,101],"identified":[67],"list":[69],"novel":[71],"features,":[72],"like":[73],"assessing":[74],"web":[75],"software,":[76],"likely":[79],"get":[81],"more":[82],"research":[83],"attention":[84],"in":[85,134],"future.":[87],"As":[88,109],"conclusion,":[90],"we":[91,112],"state":[92],"too":[94],"many":[95],"new":[96],"systems":[97,118,127],"developed,":[99],"but":[100],"acknowledge":[102],"current":[104],"reasons":[105],"phenomenon.":[108],"one":[110],"solution":[111],"encourage":[113],"opening":[114],"up":[115],"existing":[117],"joining":[120],"efforts":[121],"on":[122],"developing":[123],"those":[124],"further.":[125],"Selected":[126],"our":[129],"survey":[130],"briefly":[132],"described":[133],"Appendix":[135],"A.":[136]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":29},{"year":2023,"cited_by_count":51},{"year":2022,"cited_by_count":36},{"year":2021,"cited_by_count":35},{"year":2020,"cited_by_count":33},{"year":2019,"cited_by_count":78},{"year":2018,"cited_by_count":42},{"year":2017,"cited_by_count":40},{"year":2016,"cited_by_count":42},{"year":2015,"cited_by_count":51},{"year":2014,"cited_by_count":25},{"year":2013,"cited_by_count":31},{"year":2012,"cited_by_count":16}],"updated_date":"2026-06-23T13:55:30.953635","created_date":"2025-10-10T00:00:00"}