{"id":"https://openalex.org/W3163287068","doi":"https://doi.org/10.1145/3445983","title":"CSF: Formative Feedback in Autograding","display_name":"CSF: Formative Feedback in Autograding","publication_year":2021,"publication_date":"2021-05-10","ids":{"openalex":"https://openalex.org/W3163287068","doi":"https://doi.org/10.1145/3445983","mag":"3163287068"},"language":"en","primary_location":{"id":"doi:10.1145/3445983","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3445983","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070496918","display_name":"Georgiana Haldeman","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Georgiana Haldeman","raw_affiliation_strings":["Rutgers University, Computer Science Department, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Computer Science Department, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006699725","display_name":"Monica Babe\u015f-Vroman","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Monica Babe\u015f-Vroman","raw_affiliation_strings":["Rutgers University, Computer Science Department, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Computer Science Department, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035777090","display_name":"Andrew Tjang","orcid":null},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Tjang","raw_affiliation_strings":["Rutgers University, Computer Science Department, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Computer Science Department, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101643264","display_name":"Thu D. Nguyen","orcid":"https://orcid.org/0000-0003-2843-1796"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thu D. Nguyen","raw_affiliation_strings":["Rutgers University, Computer Science Department, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Computer Science Department, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5070496918"],"corresponding_institution_ids":["https://openalex.org/I102322142"],"apc_list":null,"apc_paid":null,"fwci":2.936,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.91406899,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"21","issue":"3","first_page":"1","last_page":"30"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9818999767303467,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/formative-assessment","display_name":"Formative assessment","score":0.8900407552719116},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8000465631484985},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.628616452217102},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5777072906494141},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4950355589389801},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.44101274013519287},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.3738836348056793},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3058234751224518},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2973538041114807},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.10194867849349976}],"concepts":[{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.8900407552719116},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8000465631484985},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.628616452217102},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5777072906494141},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4950355589389801},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.44101274013519287},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3738836348056793},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3058234751224518},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2973538041114807},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.10194867849349976},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3445983","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3445983","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8799999952316284,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W1492802215","https://openalex.org/W1791587663","https://openalex.org/W1972087783","https://openalex.org/W1981759353","https://openalex.org/W2028540741","https://openalex.org/W2054285698","https://openalex.org/W2076771354","https://openalex.org/W2135995390","https://openalex.org/W2149957844","https://openalex.org/W2151828850","https://openalex.org/W2161052636","https://openalex.org/W2179578783","https://openalex.org/W2241837413","https://openalex.org/W2246775628","https://openalex.org/W2288486118","https://openalex.org/W2299643279","https://openalex.org/W2344752727","https://openalex.org/W2465021265","https://openalex.org/W2486590439","https://openalex.org/W2512434199","https://openalex.org/W2605403059","https://openalex.org/W2611588359","https://openalex.org/W2798812031","https://openalex.org/W2966150097","https://openalex.org/W3003163926","https://openalex.org/W3009805852","https://openalex.org/W3177033768","https://openalex.org/W4249812482"],"related_works":["https://openalex.org/W2365213443","https://openalex.org/W2166247150","https://openalex.org/W4243252198","https://openalex.org/W2109507516","https://openalex.org/W2020107756","https://openalex.org/W2040397200","https://openalex.org/W3013319096","https://openalex.org/W2734792348","https://openalex.org/W2384847609","https://openalex.org/W2787286265"],"abstract_inverted_index":{"Autograding":[0],"systems":[1],"are":[2,60,90,184,194,220,226,235],"being":[3],"increasingly":[4],"deployed":[5],"to":[6,33,62,92,130,138,150,156,223,243,291,295],"meet":[7],"the":[8,45,53,68,71,85,94,119,122,128,163,187,200,207,240,252,259,263,267,272,275,298,302],"challenges":[9],"of":[10,47,55,70,75,100,121,202,239,258,266,274,287,301],"teaching":[11],"programming":[12,38],"at":[13],"scale.":[14],"Studies":[15],"show":[16,198],"that":[17,59,74,110,162,199,248],"formative":[18,35],"feedback":[19,36,288,294],"can":[20],"greatly":[21],"help":[22,139],"novices":[23],"learn":[24,98],"programming.":[25],"This":[26],"work":[27],"extends":[28],"an":[29,64,210],"autograder,":[30],"enabling":[31],"it":[32,249],"provide":[34],"on":[37,118,230,262],"assignment":[39,72],"submissions.":[40],"Our":[41],"methodology":[42,149],"starts":[43],"with":[44],"design":[46,69],"a":[48,76,108,115,168],"knowledge":[49],"map,":[50],"which":[51],"is":[52,214],"set":[54],"concepts":[56,132],"and":[57,73,97,133,135,144,160,174,190,289],"skills":[58,134],"necessary":[61],"complete":[63,206],"assignment,":[65,105],"followed":[66],"by":[67,271],"comprehensive":[77],"test":[78,93,123,253,276],"suite":[79],"for":[80,186],"identifying":[81],"logical":[82],"errors":[83,113,129],"in":[84,114,153],"submitted":[86],"code.":[87],"Test":[88],"cases":[89],"used":[91],"student":[95],"submissions":[96],"classes":[99],"common":[101],"errors.":[102],"For":[103],"each":[104],"we":[106,282],"train":[107],"classifier":[109],"automatically":[111,292],"categorizes":[112],"submission":[116,213],"based":[117],"outcome":[120,273],"suite.":[124],"The":[125,255],"instructor":[126],"maps":[127],"corresponding":[131],"writes":[136],"hints":[137,183,193,219,225,234],"students":[140,203,241],"find":[141,161],"their":[142,245],"misconceptions":[143],"mistakes.":[145],"We":[146,172],"apply":[147],"this":[148],"two":[151,178,188],"assignments":[152,189,208],"our":[154,279],"Introduction":[155],"Computer":[157],"Science":[158],"course":[159],"automatic":[164],"error":[165],"categorization":[166],"has":[167],"90%":[169],"average":[170],"accuracy.":[171],"report":[173],"compare":[175],"data":[176],"from":[177],"semesters,":[179],"one":[180,191],"semester":[181],"when":[182,192,218,224,233],"given":[185,221],"not":[195,227],"given.":[196,228],"Results":[197],"percentage":[201],"who":[204],"successfully":[205],"after":[209],"initial":[211,256],"erroneous":[212],"three":[215],"times":[216],"greater":[217],"compared":[222],"However,":[229],"average,":[231],"even":[232],"provided,":[236],"almost":[237],"half":[238],"fail":[242],"correct":[244],"code":[246],"so":[247],"passes":[250],"all":[251],"cases.":[254,277],"implementation":[257],"framework":[260],"focuses":[261],"functional":[264],"correctness":[265],"programs":[268],"as":[269],"reflected":[270],"In":[278],"future":[280],"work,":[281],"will":[283],"explore":[284],"other":[285],"kinds":[286],"approaches":[290],"generate":[293],"better":[296],"serve":[297],"educational":[299],"needs":[300],"students.":[303]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}