{"id":"https://openalex.org/W7128781914","doi":"https://doi.org/10.1145/3770761.3777261","title":"On the Efficacy of Using Large Language Models for Automatic Grading of CS Theory Problems","display_name":"On the Efficacy of Using Large Language Models for Automatic Grading of CS Theory Problems","publication_year":2026,"publication_date":"2026-02-13","ids":{"openalex":"https://openalex.org/W7128781914","doi":"https://doi.org/10.1145/3770761.3777261"},"language":null,"primary_location":{"id":"doi:10.1145/3770761.3777261","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3770761.3777261","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.2","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125984471","display_name":"Nikolas Dykstra","orcid":null},"institutions":[{"id":"https://openalex.org/I192545095","display_name":"United States Military Academy","ror":"https://ror.org/01jepya76","country_code":"US","type":"education","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I192545095","https://openalex.org/I4210088792"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Nikolas Dykstra","raw_affiliation_strings":["United States Military Academy, West Point, NY, USA"],"affiliations":[{"raw_affiliation_string":"United States Military Academy, West Point, NY, USA","institution_ids":["https://openalex.org/I192545095"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125933622","display_name":"Reid Wesley","orcid":null},"institutions":[{"id":"https://openalex.org/I192545095","display_name":"United States Military Academy","ror":"https://ror.org/01jepya76","country_code":"US","type":"education","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I192545095","https://openalex.org/I4210088792"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Reid Wesley","raw_affiliation_strings":["United States Military Academy, West Point, NY, USA"],"affiliations":[{"raw_affiliation_string":"United States Military Academy, West Point, NY, USA","institution_ids":["https://openalex.org/I192545095"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035179393","display_name":"Ryan E. Dougherty","orcid":"https://orcid.org/0000-0003-1739-1127"},"institutions":[{"id":"https://openalex.org/I192545095","display_name":"United States Military Academy","ror":"https://ror.org/01jepya76","country_code":"US","type":"education","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I192545095","https://openalex.org/I4210088792"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ryan E. Dougherty","raw_affiliation_strings":["United States Military Academy, West Point, NY, USA"],"affiliations":[{"raw_affiliation_string":"United States Military Academy, West Point, NY, USA","institution_ids":["https://openalex.org/I192545095"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5125984471"],"corresponding_institution_ids":["https://openalex.org/I192545095"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.74161666,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1311","last_page":"1312"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.661300003528595,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.661300003528595,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.12269999831914902,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12884","display_name":"Educational Assessment and Pedagogy","score":0.022299999371170998,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.8676999807357788},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.7196999788284302},{"id":"https://openalex.org/keywords/notation","display_name":"Notation","score":0.5684999823570251},{"id":"https://openalex.org/keywords/rule-based-machine-translation","display_name":"Rule-based machine translation","score":0.5120000243186951},{"id":"https://openalex.org/keywords/undecidable-problem","display_name":"Undecidable problem","score":0.41589999198913574},{"id":"https://openalex.org/keywords/disk-formatting","display_name":"Disk formatting","score":0.41370001435279846},{"id":"https://openalex.org/keywords/compiler","display_name":"Compiler","score":0.36550000309944153},{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.3540000021457672}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.8676999807357788},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.7196999788284302},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.704200029373169},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5893999934196472},{"id":"https://openalex.org/C45357846","wikidata":"https://www.wikidata.org/wiki/Q2001982","display_name":"Notation","level":2,"score":0.5684999823570251},{"id":"https://openalex.org/C53893814","wikidata":"https://www.wikidata.org/wiki/Q7378909","display_name":"Rule-based machine translation","level":2,"score":0.5120000243186951},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.477400004863739},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4343999922275543},{"id":"https://openalex.org/C192034797","wikidata":"https://www.wikidata.org/wiki/Q3502995","display_name":"Undecidable problem","level":3,"score":0.41589999198913574},{"id":"https://openalex.org/C88006597","wikidata":"https://www.wikidata.org/wiki/Q690117","display_name":"Disk formatting","level":2,"score":0.41370001435279846},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.36550000309944153},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.3540000021457672},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.353300005197525},{"id":"https://openalex.org/C9870796","wikidata":"https://www.wikidata.org/wiki/Q490481","display_name":"Turing","level":2,"score":0.3508000075817108},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.30820000171661377},{"id":"https://openalex.org/C29248071","wikidata":"https://www.wikidata.org/wiki/Q163310","display_name":"Turing machine","level":3,"score":0.29190000891685486},{"id":"https://openalex.org/C26022165","wikidata":"https://www.wikidata.org/wiki/Q8091","display_name":"Grammar","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C94922259","wikidata":"https://www.wikidata.org/wiki/Q33215","display_name":"Constructed language","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2572000026702881}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3770761.3777261","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3770761.3777261","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.2","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.862735390663147,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W4400267607","https://openalex.org/W4404910483"],"related_works":[],"abstract_inverted_index":{"Automating":[0],"the":[1,10,26,101,116,137],"grading":[2,36,122],"of":[3,28,80],"theoretical":[4],"computer":[5],"science":[6],"(TCS)":[7],"problems":[8],"has":[9],"potential":[11],"to":[12,55,100,108,140],"save":[13],"instructors":[14],"significant":[15],"time":[16],"while":[17],"providing":[18],"students":[19],"with":[20,87,95,124],"consistent":[21],"feedback.":[22],"This":[23],"poster":[24],"explores":[25],"efficacy":[27],"a":[29,74,78],"large":[30],"language":[31],"model":[32],"(LLM)":[33],"in":[34,131],"automatically":[35],"undergraduate-level":[37],"TCS":[38],"problems.":[39],"We":[40],"focus":[41],"on":[42],"common":[43],"topics":[44],"such":[45],"as":[46],"creating":[47],"context-free":[48],"grammars":[49,54],"and":[50,62,76,85,103,143,146],"pushdown":[51],"automata,":[52],"converting":[53],"Chomsky":[56],"Normal":[57],"Form,":[58],"designing":[59],"Turing":[60],"machines,":[61],"proving":[63],"languages":[64],"undecidable":[65],"via":[66],"reductions.":[67],"For":[68],"each":[69],"problem":[70],"type,":[71],"we":[72],"developed":[73],"rubric":[75,133],"generated":[77],"dataset":[79],"student-like":[81],"solutions,":[82],"some":[83],"correct":[84],"others":[86],"typical":[88],"student":[89],"errors.":[90,151],"The":[91],"LLM":[92,117,138],"was":[93],"tasked":[94],"scoring":[96],"these":[97],"solutions":[98],"according":[99],"rubric,":[102],"its":[104],"grades":[105],"were":[106],"compared":[107],"human":[109,121],"instructor":[110],"grades.":[111],"Our":[112],"results":[113],"show":[114],"that":[115],"can":[118],"often":[119],"mimic":[120],"trends":[123],"moderate":[125],"accuracy,":[126],"but":[127],"notable":[128],"discrepancies":[129],"arise":[130],"specific":[132],"categories.":[134],"In":[135],"particular,":[136],"tends":[139],"over-penalize":[141],"formatting":[142],"notation":[144],"issues":[145],"sometimes":[147],"overlooks":[148],"deeper":[149],"logical":[150]},"counts_by_year":[],"updated_date":"2026-02-15T05:58:04.055770","created_date":"2026-02-14T00:00:00"}