{"id":"https://openalex.org/W4415622408","doi":"https://doi.org/10.1145/3773911","title":"A Survey on Feedback Types in Automated Programming Assessment Systems","display_name":"A Survey on Feedback Types in Automated Programming Assessment Systems","publication_year":2025,"publication_date":"2025-10-28","ids":{"openalex":"https://openalex.org/W4415622408","doi":"https://doi.org/10.1145/3773911"},"language":"en","primary_location":{"id":"doi:10.1145/3773911","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3773911","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5095024525","display_name":"Eduard Frankford","orcid":"https://orcid.org/0009-0005-5959-4936"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Eduard Frankford","raw_affiliation_strings":["Department of Computer Science, University of Innsbruck, Innsbruck, Austria","University of Innsbruck, Department of Computer Science, Austria"],"raw_orcid":"https://orcid.org/0009-0005-5959-4936","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Innsbruck, Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]},{"raw_affiliation_string":"University of Innsbruck, Department of Computer Science, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092130694","display_name":"Tobias Antensteiner","orcid":"https://orcid.org/0000-0001-5513-1073"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Tobias Antensteiner","raw_affiliation_strings":["Department of Computer Science, University of Innsbruck, Innsbruck, Austria","University of Innsbruck, Department of Computer Science, Austria"],"raw_orcid":"https://orcid.org/0000-0001-5513-1073","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Innsbruck, Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]},{"raw_affiliation_string":"University of Innsbruck, Department of Computer Science, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005248504","display_name":"Michael Vierhauser","orcid":"https://orcid.org/0000-0003-2672-9230"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Michael Vierhauser","raw_affiliation_strings":["Department of Computer Science, University of Innsbruck, Innsbruck, Austria","University of Innsbruck, Department of Computer Science, Austria"],"raw_orcid":"https://orcid.org/0000-0003-2672-9230","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Innsbruck, Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]},{"raw_affiliation_string":"University of Innsbruck, Department of Computer Science, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039557173","display_name":"Clemens Sauerwein","orcid":"https://orcid.org/0009-0009-9464-5080"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Clemens Sauerwein","raw_affiliation_strings":["Department of Computer Science, University of Innsbruck, Innsbruck, Austria","University of Innsbruck, Department of Computer Science, Austria"],"raw_orcid":"https://orcid.org/0009-0009-9464-5080","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Innsbruck, Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]},{"raw_affiliation_string":"University of Innsbruck, Department of Computer Science, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005370502","display_name":"Vivien Wallner","orcid":"https://orcid.org/0000-0002-7685-6273"},"institutions":[{"id":"https://openalex.org/I161878677","display_name":"Austrian Research Institute for Artificial Intelligence","ror":"https://ror.org/04j47vk14","country_code":"AT","type":"facility","lineage":["https://openalex.org/I161878677","https://openalex.org/I4210107880"]},{"id":"https://openalex.org/I182212641","display_name":"University of Salzburg","ror":"https://ror.org/05gs8cd61","country_code":"AT","type":"education","lineage":["https://openalex.org/I182212641"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Vivien Wallner","raw_affiliation_strings":["Department of Artificial Intelligence and Human Interfaces, University of Salzburg, Salzburg, Austria","University of Salzburg, Department of Artificial Intelligence and Human Interfaces, Austria"],"raw_orcid":"https://orcid.org/0000-0002-7685-6273","affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence and Human Interfaces, University of Salzburg, Salzburg, Austria","institution_ids":["https://openalex.org/I182212641"]},{"raw_affiliation_string":"University of Salzburg, Department of Artificial Intelligence and Human Interfaces, Austria","institution_ids":["https://openalex.org/I161878677"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002934401","display_name":"Iris Groher","orcid":"https://orcid.org/0000-0003-0905-6791"},"institutions":[{"id":"https://openalex.org/I121883995","display_name":"Johannes Kepler University of Linz","ror":"https://ror.org/052r2xn60","country_code":"AT","type":"education","lineage":["https://openalex.org/I121883995"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Iris Groher","raw_affiliation_strings":["Institute of Business Informatics\u2013Software Engineering, Johannes Kepler University Linz, Linz, Austria","Johannes Kepler University Linz, Institute of Business Informatics - Software Engineering, Austria"],"raw_orcid":"https://orcid.org/0000-0003-0905-6791","affiliations":[{"raw_affiliation_string":"Institute of Business Informatics\u2013Software Engineering, Johannes Kepler University Linz, Linz, Austria","institution_ids":["https://openalex.org/I121883995"]},{"raw_affiliation_string":"Johannes Kepler University Linz, Institute of Business Informatics - Software Engineering, Austria","institution_ids":["https://openalex.org/I121883995"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082242659","display_name":"Reinhold Pl\u00f6sch","orcid":"https://orcid.org/0000-0002-4659-2385"},"institutions":[{"id":"https://openalex.org/I121883995","display_name":"Johannes Kepler University of Linz","ror":"https://ror.org/052r2xn60","country_code":"AT","type":"education","lineage":["https://openalex.org/I121883995"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Reinhold Pl\u00f6sch","raw_affiliation_strings":["Institute of Business Informatics\u2013Software Engineering, Johannes Kepler University\rLinz, Linz, Austria","Johannes Kepler University Linz, Institute of Business Informatics - Software Engineering, Austria"],"raw_orcid":"https://orcid.org/0000-0002-4659-2385","affiliations":[{"raw_affiliation_string":"Institute of Business Informatics\u2013Software Engineering, Johannes Kepler University\rLinz, Linz, Austria","institution_ids":["https://openalex.org/I121883995"]},{"raw_affiliation_string":"Johannes Kepler University Linz, Institute of Business Informatics - Software Engineering, Austria","institution_ids":["https://openalex.org/I121883995"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037343322","display_name":"Ruth Breu","orcid":"https://orcid.org/0000-0001-7093-4341"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Ruth Breu","raw_affiliation_strings":["Department of Computer Science, University of Innsbruck, Innsbruck, Austria","University of Innsbruck, Department of Computer Science, Austria"],"raw_orcid":"https://orcid.org/0000-0001-7093-4341","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Innsbruck, Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]},{"raw_affiliation_string":"University of Innsbruck, Department of Computer Science, Austria","institution_ids":["https://openalex.org/I190249584"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.913,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.94739957,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":"26","issue":"1","first_page":"1","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.48350000381469727,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.48350000381469727,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.10620000213384628,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.03150000050663948,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/unit-testing","display_name":"Unit testing","score":0.6699000000953674},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.5609999895095825},{"id":"https://openalex.org/keywords/compiler","display_name":"Compiler","score":0.5199999809265137},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.4896000027656555},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.47519999742507935},{"id":"https://openalex.org/keywords/unit","display_name":"Unit (ring theory)","score":0.4708000123500824},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.46959999203681946}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8172000050544739},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.6699000000953674},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.5609999895095825},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.5199999809265137},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.4896000027656555},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.47519999742507935},{"id":"https://openalex.org/C122637931","wikidata":"https://www.wikidata.org/wiki/Q118084","display_name":"Unit (ring theory)","level":2,"score":0.4708000123500824},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.46959999203681946},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.4478999972343445},{"id":"https://openalex.org/C2779308522","wikidata":"https://www.wikidata.org/wiki/Q843958","display_name":"Digitization","level":2,"score":0.4316999912261963},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4099999964237213},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.36910000443458557},{"id":"https://openalex.org/C2776836400","wikidata":"https://www.wikidata.org/wiki/Q31087","display_name":"Calculator","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C105776082","wikidata":"https://www.wikidata.org/wiki/Q617473","display_name":"Likert scale","level":2,"score":0.3019999861717224},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.29600000381469727},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C33857546","wikidata":"https://www.wikidata.org/wiki/Q80006","display_name":"Computer programming","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2750999927520752},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26969999074935913},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2687999904155731},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.26080000400543213}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3773911","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3773911","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W1570528029","https://openalex.org/W1873022895","https://openalex.org/W1983517965","https://openalex.org/W1988643350","https://openalex.org/W2012312630","https://openalex.org/W2014538668","https://openalex.org/W2041282815","https://openalex.org/W2070405152","https://openalex.org/W2110661236","https://openalex.org/W2130625835","https://openalex.org/W2293412016","https://openalex.org/W2750631396","https://openalex.org/W2807648480","https://openalex.org/W2888808568","https://openalex.org/W3011884107","https://openalex.org/W3022044005","https://openalex.org/W3043237517","https://openalex.org/W3043358777","https://openalex.org/W3086838501","https://openalex.org/W3122656795","https://openalex.org/W3183577725","https://openalex.org/W4213421742","https://openalex.org/W4381586920"],"related_works":[],"abstract_inverted_index":{"With":[0,118],"the":[1,16,104,119,208],"recent":[2,127],"rapid":[3],"increase":[4],"in":[5,26,126,148,159,236],"digitization":[6],"across":[7],"all":[8],"major":[9],"industries,":[10],"acquiring":[11],"programming":[12,20,29,237],"skills":[13],"has":[14,23],"increased":[15],"demand":[17],"for":[18,55,98],"introductory":[19],"courses.":[21],"This":[22],"further":[24],"resulted":[25],"universities":[27],"integrating":[28],"courses":[30],"into":[31],"a":[32,165],"wide":[33],"range":[34],"of":[35,48,69,108,110,121],"curricula,":[36],"including":[37],"not":[38],"only":[39],"technical":[40],"studies":[41],"but":[42],"also":[43],"business":[44],"and":[45,58,65,80,87,106,140,154,186,193,224,232],"management":[46],"fields":[47],"study.":[49],"Consequently,":[50],"additional":[51],"resources":[52],"are":[53,150,158],"needed":[54],"teaching,":[56],"grading,":[57],"tutoring":[59],"students":[60,171,202],"with":[61,84,168],"diverse":[62],"educational":[63],"backgrounds":[64],"skills.":[66],"As":[67],"part":[68],"this,":[70],"Automated":[71],"Programming":[72],"Assessment":[73],"Systems":[74],"(APASs)":[75],"have":[76,131,163],"emerged,":[77],"providing":[78],"scalable":[79],"high-quality":[81],"assessment":[82],"systems":[83],"efficient":[85],"evaluation":[86],"instant":[88],"feedback.":[89],"Commonly,":[90],"APASs":[91,149],"heavily":[92],"rely":[93],"on":[94,195],"predefined":[95],"unit":[96,222],"tests":[97,223],"generating":[99],"feedback,":[100],"which":[101],"often":[102],"limits":[103],"scope":[105],"level":[107],"detail":[109],"feedback":[111,138,146,212,230],"that":[112,200],"can":[113,136],"be":[114],"provided":[115],"to":[116,214,227],"students.":[117],"rise":[120],"Large":[122],"Language":[123],"Models":[124],"(LLMs)":[125],"years,":[128],"new":[129],"opportunities":[130],"emerged":[132],"as":[133,207],"these":[134],"technologies":[135],"enhance":[137],"quality":[139,192],"personalization.":[141],"To":[142],"investigate":[143],"how":[144,155],"different":[145,174],"mechanisms":[147,231],"perceived":[151,191],"by":[152],"students,":[153],"effective":[156],"they":[157],"supporting":[160],"problem-solving,":[161],"we":[162,177],"conducted":[164],"large-scale":[166],"study":[167],"over":[169],"200":[170],"from":[172],"2":[173],"universities.":[175],"Specifically,":[176],"compare":[178],"baseline":[179],"Compiler":[180],"Feedback,":[181,185],"standard":[182],"Unit":[183,204],"Test":[184,205],"advanced":[187],"LLM-based":[188],"Feedback":[189,206],"regarding":[190],"impact":[194],"student":[196],"performance.":[197],"Results":[198],"indicate":[199],"while":[201],"rate":[203],"most":[209],"helpful,":[210],"AI-generated":[211],"leads":[213],"significantly":[215],"better":[216],"performances.":[217],"These":[218],"findings":[219],"suggest":[220],"combining":[221],"AI-driven":[225],"guidance":[226],"optimize":[228],"automated":[229],"improve":[233],"learning":[234],"outcomes":[235],"education.":[238]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-28T00:00:00"}