{"id":"https://openalex.org/W2963666583","doi":"https://doi.org/10.1609/aaai.v33i01.3301782","title":"Zero Shot Learning for Code Education: Rubric Sampling with Deep Learning Inference","display_name":"Zero Shot Learning for Code Education: Rubric Sampling with Deep Learning Inference","publication_year":2019,"publication_date":"2019-07-17","ids":{"openalex":"https://openalex.org/W2963666583","doi":"https://doi.org/10.1609/aaai.v33i01.3301782","mag":"2963666583"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v33i01.3301782","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.3301782","pdf_url":"https://aaai.org/ojs/index.php/AAAI/article/download/3857/3735","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://aaai.org/ojs/index.php/AAAI/article/download/3857/3735","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109077106","display_name":"Mike Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mike Wu","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029368853","display_name":"Milan Mosse","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Milan Mosse","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001961716","display_name":"Noah D. Goodman","orcid":"https://orcid.org/0000-0002-9176-8802"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Noah Goodman","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074969309","display_name":"Chris Piech","orcid":"https://orcid.org/0000-0001-5140-0467"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Piech","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":40,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":"01","first_page":"782","last_page":"790"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.9320846796035767},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7502127885818481},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.5257773399353027},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.49390918016433716},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4889880418777466},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.47980907559394836},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.460397869348526},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.4413992762565613},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4244825839996338},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.3658422529697418},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1270926296710968},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09261497855186462},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09050330519676208},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.08839333057403564}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9320846796035767},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7502127885818481},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.5257773399353027},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.49390918016433716},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4889880418777466},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.47980907559394836},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.460397869348526},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4413992762565613},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4244825839996338},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3658422529697418},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1270926296710968},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09261497855186462},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09050330519676208},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.08839333057403564},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v33i01.3301782","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.3301782","pdf_url":"https://aaai.org/ojs/index.php/AAAI/article/download/3857/3735","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/3857","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/3857","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v33i01.3301782","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.3301782","pdf_url":"https://aaai.org/ojs/index.php/AAAI/article/download/3857/3735","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7799999713897705}],"awards":[{"id":"https://openalex.org/G4713059963","display_name":null,"funder_award_id":"FA8750","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2963666583.pdf","grobid_xml":"https://content.openalex.org/works/W2963666583.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W2023164354","https://openalex.org/W2042742211","https://openalex.org/W2064430731","https://openalex.org/W2083590325","https://openalex.org/W2129740354","https://openalex.org/W2187089797","https://openalex.org/W2194321275","https://openalex.org/W2210838531","https://openalex.org/W2341162834","https://openalex.org/W2401080241","https://openalex.org/W2596367596","https://openalex.org/W2771764532","https://openalex.org/W2772366472","https://openalex.org/W2786541991","https://openalex.org/W2786623891","https://openalex.org/W2788286534","https://openalex.org/W2805937758","https://openalex.org/W2962858109","https://openalex.org/W2962907997","https://openalex.org/W2963052338","https://openalex.org/W2963545832","https://openalex.org/W2963948515","https://openalex.org/W2963960318","https://openalex.org/W2964248207","https://openalex.org/W4253804874","https://openalex.org/W4312501370","https://openalex.org/W6671734478","https://openalex.org/W6746524570"],"related_works":["https://openalex.org/W1987813225","https://openalex.org/W4390610055","https://openalex.org/W4379053243","https://openalex.org/W3021020626","https://openalex.org/W2955201676","https://openalex.org/W2153005530","https://openalex.org/W2330342969","https://openalex.org/W2777782248","https://openalex.org/W2080627647","https://openalex.org/W4213362990"],"abstract_inverted_index":{"In":[0,60,76],"modern":[1],"computer":[2],"science":[3],"education,":[4],"massive":[5],"open":[6],"online":[7],"courses":[8],"(MOOCs)":[9],"log":[10],"thousands":[11],"of":[12,14,33,134,147],"hours":[13],"data":[15,72,164],"about":[16,54],"how":[17],"students":[18,50,103],"solve":[19],"coding":[20],"challenges.":[21],"Being":[22],"so":[23],"rich":[24],"in":[25,144],"data,":[26],"these":[27],"platforms":[28],"have":[29,69],"garnered":[30],"the":[31,34,88,101,145,148,177],"interest":[32],"machine":[35],"learning":[36,151],"community,":[37],"with":[38,110,131],"many":[39],"new":[40],"algorithms":[41,116],"attempting":[42],"to":[43,47,86,96,156],"autonomously":[44],"provide":[45,97],"feedback":[46,91,99,130],"help":[48],"future":[49],"learn.":[51],"But":[52],"what":[53],"those":[55],"first":[56,102],"hundred":[57],"thousand":[58],"students?":[59],"most":[61],"educational":[62],"contexts":[63],"(i.e.":[64],"classrooms),":[65],"assignments":[66],"do":[67],"not":[68],"enough":[70],"historical":[71],"for":[73,100],"supervised":[74],"learning.":[75],"this":[77],"paper,":[78],"we":[79],"introduce":[80],"a":[81,135,141,172],"human-in-the-loop":[82],"\u201crubric":[83],"sampling\u201d":[84],"approach":[85],"tackle":[87],"\u201czero":[89],"shot\u201d":[90],"challenge.":[92],"We":[93,167],"are":[94],"able":[95],"autonomous":[98],"working":[104],"on":[105,171],"an":[106],"introductory":[107],"programming":[108,180],"assignment":[109,161],"accuracy":[111],"that":[112],"substantially":[113],"outperforms":[114],"data-hungry":[115],"and":[117,138],"approaches":[118],"human":[119],"level":[120],"fidelity.":[121],"Rubric":[122],"sampling":[123,155],"requires":[124],"minimal":[125],"teacher":[126],"effort,":[127],"can":[128,139],"associate":[129],"specific":[132,162],"parts":[133],"student\u2019s":[136,142],"solution":[137],"articulate":[140],"misconceptions":[143],"language":[146],"instructor.":[149],"Deep":[150],"inference":[152],"enables":[153],"rubric":[154],"further":[157],"improve":[158],"as":[159],"more":[160],"student":[163],"is":[165],"acquired.":[166],"demonstrate":[168],"our":[169],"results":[170],"novel":[173],"dataset":[174],"from":[175],"Code.org,":[176],"world\u2019s":[178],"largest":[179],"education":[181],"platform.":[182]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}