{"id":"https://openalex.org/W4392542471","doi":"https://doi.org/10.1145/3626252.3630759","title":"A Fast and Accurate Machine Learning Autograder for the Breakout Assignment","display_name":"A Fast and Accurate Machine Learning Autograder for the Breakout Assignment","publication_year":2024,"publication_date":"2024-03-07","ids":{"openalex":"https://openalex.org/W4392542471","doi":"https://doi.org/10.1145/3626252.3630759"},"language":"en","primary_location":{"id":"doi:10.1145/3626252.3630759","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626252.3630759","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th ACM Technical Symposium on Computer Science Education V. 1","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045100443","display_name":"Evan Zheran Liu","orcid":"https://orcid.org/0009-0001-4411-7599"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Evan Zheran Liu","raw_affiliation_strings":["Stanford University & Imbue, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University & Imbue, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101413641","display_name":"David D. Yuan","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Yuan","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100713134","display_name":"Ahmed Ahmed","orcid":"https://orcid.org/0009-0003-2275-3230"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed Ahmed","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094087820","display_name":"Elyse Cornwall","orcid":"https://orcid.org/0009-0004-5418-9993"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Elyse Cornwall","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061308206","display_name":"Juliette Woodrow","orcid":"https://orcid.org/0009-0006-8097-093X"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juliette Woodrow","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074014720","display_name":"Kaylee Burns","orcid":"https://orcid.org/0000-0001-5713-2774"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kaylee Burns","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018535992","display_name":"Allen Nie","orcid":"https://orcid.org/0000-0001-6483-2111"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Allen Nie","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084989076","display_name":"Emma Brunskill","orcid":"https://orcid.org/0000-0002-3971-7127"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Emma Brunskill","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074969309","display_name":"Chris Piech","orcid":"https://orcid.org/0000-0001-5140-0467"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Piech","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005431772","display_name":"Chelsea Finn","orcid":"https://orcid.org/0000-0001-6298-0874"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chelsea Finn","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5045100443"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":0.4029,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.67391217,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"736","last_page":"742"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.984499990940094,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9605000019073486,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/breakout","display_name":"Breakout","score":0.8917678594589233},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.8340930342674255},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7812457084655762},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.6068805456161499},{"id":"https://openalex.org/keywords/unit-testing","display_name":"Unit testing","score":0.5014798641204834},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47642573714256287},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.4583359360694885},{"id":"https://openalex.org/keywords/paddle","display_name":"Paddle","score":0.43743178248405457},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3823067545890808},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3314833641052246},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.24326637387275696},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.23313656449317932},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09799334406852722},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.08660939335823059}],"concepts":[{"id":"https://openalex.org/C2778091849","wikidata":"https://www.wikidata.org/wiki/Q4959649","display_name":"Breakout","level":2,"score":0.8917678594589233},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.8340930342674255},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7812457084655762},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.6068805456161499},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.5014798641204834},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47642573714256287},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4583359360694885},{"id":"https://openalex.org/C165150422","wikidata":"https://www.wikidata.org/wiki/Q449573","display_name":"Paddle","level":2,"score":0.43743178248405457},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3823067545890808},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3314833641052246},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.24326637387275696},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.23313656449317932},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09799334406852722},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.08660939335823059},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626252.3630759","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626252.3630759","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th ACM Technical Symposium on Computer Science Education V. 1","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.699999988079071}],"awards":[{"id":"https://openalex.org/G1815357327","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320309949","funder_display_name":"Canadian Institute for Advanced Research"},{"id":"https://openalex.org/G2499887605","display_name":null,"funder_award_id":"DGE-1656518","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G320138474","display_name":null,"funder_award_id":"-1656518","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3944636959","display_name":null,"funder_award_id":"Hoffman Yee Award","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"},{"id":"https://openalex.org/G5267932457","display_name":null,"funder_award_id":"1656518","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5357174734","display_name":null,"funder_award_id":"DGE-1656518","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"},{"id":"https://openalex.org/G5614971186","display_name":null,"funder_award_id":"Fellow","funder_id":"https://openalex.org/F4320309949","funder_display_name":"Canadian Institute for Advanced Research"},{"id":"https://openalex.org/G6894402473","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309949","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W398859631","https://openalex.org/W1968088872","https://openalex.org/W2012312630","https://openalex.org/W2054673081","https://openalex.org/W2076771354","https://openalex.org/W2125344860","https://openalex.org/W2246775628","https://openalex.org/W2294064749","https://openalex.org/W2524230755","https://openalex.org/W2605403059","https://openalex.org/W2607384194","https://openalex.org/W2788496235","https://openalex.org/W2789176835","https://openalex.org/W2915075219","https://openalex.org/W2970680470","https://openalex.org/W3006614568","https://openalex.org/W3009731884","https://openalex.org/W3046760799","https://openalex.org/W3128345447","https://openalex.org/W3135705253","https://openalex.org/W3137081879","https://openalex.org/W3168265850","https://openalex.org/W4213421742","https://openalex.org/W4240488395","https://openalex.org/W4319996980","https://openalex.org/W4390606952"],"related_works":["https://openalex.org/W2494799991","https://openalex.org/W2369586047","https://openalex.org/W2360720760","https://openalex.org/W2089901852","https://openalex.org/W2033715836","https://openalex.org/W2380756105","https://openalex.org/W2383313205","https://openalex.org/W2354342024","https://openalex.org/W2370950893","https://openalex.org/W2017605127"],"abstract_inverted_index":{"In":[0,139],"this":[1,119],"paper,":[2],"we":[3,146],"detail":[4],"the":[5,16,21,47,71,114,177,183],"successful":[6],"deployment":[7,172],"of":[8,37,86,101,113,134,176,185],"a":[9,34,38,42,65,104,127],"machine":[10,198],"learning":[11,129,199],"autograder":[12,117,152],"that":[13,44,148],"significantly":[14],"decreases":[15],"grading":[17,67,75,102,154,161,186],"labor":[18],"required":[19],"in":[20,103,173],"Breakout":[22],"computer":[23,60],"science":[24,61],"assignment.":[25,115,178],"This":[26,96],"assignment":[27],"-":[28,52],"which":[29],"tasks":[30],"students":[31,57],"with":[32,58,126,143],"programming":[33],"game":[35,91,125],"consisting":[36],"controllable":[39],"paddle":[40,48],"and":[41,80,108,131],"ball":[43],"bounces":[45],"off":[46],"to":[49,70,92,98,137],"break":[50],"bricks":[51],"is":[53],"popular":[54],"for":[55,94],"engaging":[56],"introductory":[59],"concepts,":[62],"but":[63],"creates":[64],"large":[66],"burden.":[68],"Due":[69],"game's":[72],"interactive":[73,188],"nature,":[74],"defies":[76],"traditional":[77],"unit":[78],"tests":[79],"instead":[81],"typically":[82],"requires":[83],"8+":[84],"minutes":[85],"manually":[87],"playing":[88,122],"each":[89,123],"student's":[90,124],"search":[93],"bugs.":[95],"amounts":[97],"45+":[99],"hours":[100,169],"standard":[105],"course":[106],"offering":[107],"prevents":[109],"further":[110,181],"widespread":[111],"adoption":[112],"Our":[116,179],"alleviates":[118],"burden":[120],"by":[121,156,163],"reinforcement":[128],"agent":[130],"providing":[132],"videos":[133],"discovered":[135],"bugs":[136],"instructors.":[138],"an":[140],"A/B":[141],"test":[142],"manual":[144],"grading,":[145],"find":[147],"our":[149,171],"human-in-the-loop":[150],"AI":[151],"reduces":[153],"time":[155],"44%,":[157],"while":[158],"slightly":[159],"improving":[160],"accuracy":[162],"6%,":[164],"ultimately":[165],"saving":[166],"roughly":[167],"30":[168],"over":[170],"two":[174],"offerings":[175],"results":[180],"suggest":[182],"practicality":[184],"other":[187,191],"assignments":[189],"(e.g.,":[190],"games":[192],"or":[193],"building":[194],"websites)":[195],"via":[196],"similar":[197],"techniques.":[200],"Live":[201],"demo":[202],"at":[203],"https://ezliu.github.io/breakoutgrader.":[204]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}