{"id":"https://openalex.org/W2788496235","doi":"https://doi.org/10.1145/3159450.3159502","title":"Providing Meaningful Feedback for Autograding of Programming Assignments","display_name":"Providing Meaningful Feedback for Autograding of Programming Assignments","publication_year":2018,"publication_date":"2018-02-21","ids":{"openalex":"https://openalex.org/W2788496235","doi":"https://doi.org/10.1145/3159450.3159502","mag":"2788496235"},"language":"en","primary_location":{"id":"doi:10.1145/3159450.3159502","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3159450.3159502","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070496918","display_name":"Georgiana Haldeman","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Georgiana Haldeman","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035777090","display_name":"Andrew Tjang","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Tjang","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006699725","display_name":"Monica Babe\u015f-Vroman","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Monica Babe\u015f-Vroman","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020773437","display_name":"Stephen Bartos","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephen Bartos","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086669933","display_name":"Jay Shah","orcid":"https://orcid.org/0000-0002-1187-5430"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jay Shah","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017318216","display_name":"Danielle Yucht","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danielle Yucht","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101643264","display_name":"Thu D. Nguyen","orcid":"https://orcid.org/0000-0003-2843-1796"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thu D. Nguyen","raw_affiliation_strings":["Rutgers University, Piscataway, NH, USA"],"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NH, USA","institution_ids":["https://openalex.org/I102322142"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5070496918"],"corresponding_institution_ids":["https://openalex.org/I102322142"],"apc_list":null,"apc_paid":null,"fwci":4.727,"has_fulltext":false,"cited_by_count":46,"citation_normalized_percentile":{"value":0.94635378,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"278","last_page":"283"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8518210649490356},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.7109519839286804},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.6637095808982849},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.6217077970504761},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.47758129239082336},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.47631916403770447},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.410768061876297},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32429349422454834},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.23496487736701965}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8518210649490356},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.7109519839286804},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.6637095808982849},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.6217077970504761},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.47758129239082336},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.47631916403770447},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.410768061876297},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32429349422454834},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.23496487736701965},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3159450.3159502","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3159450.3159502","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8799999952316284}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W1492802215","https://openalex.org/W2012312630","https://openalex.org/W2028540741","https://openalex.org/W2089010774","https://openalex.org/W2129740354","https://openalex.org/W2246775628","https://openalex.org/W2277809516","https://openalex.org/W2288486118","https://openalex.org/W2308618763","https://openalex.org/W2338451774","https://openalex.org/W2396992199","https://openalex.org/W2464228044","https://openalex.org/W2465021265","https://openalex.org/W2512434199","https://openalex.org/W2534769760","https://openalex.org/W2605403059","https://openalex.org/W3003163926","https://openalex.org/W4249812482","https://openalex.org/W4253641612","https://openalex.org/W6694706704"],"related_works":["https://openalex.org/W2165912799","https://openalex.org/W2735662278","https://openalex.org/W2382615723","https://openalex.org/W4231704780","https://openalex.org/W4311804456","https://openalex.org/W1987484445","https://openalex.org/W2623658258","https://openalex.org/W1969219540","https://openalex.org/W2143413548","https://openalex.org/W2370459448"],"abstract_inverted_index":{"Autograding":[0],"systems":[1],"are":[2,55,180],"increasingly":[3],"being":[4],"deployed":[5,176],"to":[6,22,41,58,61,70,82,89,102,108,121],"meet":[7],"the":[8,33,36,46,80,100,119,134,142,153,167],"challenge":[9],"of":[10,64,118,141,166],"teaching":[11],"programming":[12,43],"at":[13],"scale.":[14],"We":[15,97,112,131,150],"propose":[16],"a":[17,50,147],"methodology":[18,30,101],"for":[19,26,127,156,162],"extending":[20],"autograders":[21],"provide":[23],"meaningful":[24],"feedback":[25,185],"incorrect":[27],"programs.":[28],"Our":[29],"starts":[31],"with":[32],"instructor":[34,78],"identifying":[35],"concepts":[37,83],"and":[38,48,67,84,86,95,124,138,179,184,188],"skills":[39,85],"important":[40],"each":[42],"assignment,":[44,47],"designing":[45,49],"comprehensive":[51],"test":[52],"suite.":[53],"Tests":[54],"then":[56],"applied":[57,99],"code":[59],"submissions":[60,114,145,158,183],"learn":[62],"classes":[63],"common":[65,129],"errors":[66,73,81],"produce":[68],"classifiers":[69,123],"automatically":[71],"categorize":[72],"in":[74],"future":[75],"submissions.":[76],"The":[77],"maps":[79],"writes":[87],"hints":[88,126,143,154,178],"help":[90],"students":[91,187],"find":[92],"their":[93],"misconceptions":[94],"mistakes.":[96],"have":[98,175],"two":[103],"assignments":[104],"from":[105,115,146,186],"our":[106,177],"Introduction":[107],"Computer":[109],"Science":[110],"course.":[111],"used":[113],"one":[116],"semester":[117],"class":[120],"build":[122],"write":[125],"observed":[128],"errors.":[130],"manually":[132],"validated":[133],"automatic":[135],"error":[136],"categorization":[137],"potential":[139],"usefulness":[140],"using":[144],"second":[148],"semester.":[149],"found":[151],"that":[152],"given":[155],"erroneous":[157],"should":[159],"be":[160],"helpful":[161],"96%":[163],"or":[164],"more":[165],"cases.":[168],"Based":[169],"on":[170],"these":[171],"promising":[172],"results,":[173],"we":[174],"currently":[181],"collecting":[182],"instructors.":[189]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}