{"id":"https://openalex.org/W7128796115","doi":"https://doi.org/10.1145/3770761.3777018","title":"LLMTutorBench: A Benchmark for University-level TCS AI Tutoring Systems","display_name":"LLMTutorBench: A Benchmark for University-level TCS AI Tutoring Systems","publication_year":2026,"publication_date":"2026-02-13","ids":{"openalex":"https://openalex.org/W7128796115","doi":"https://doi.org/10.1145/3770761.3777018"},"language":null,"primary_location":{"id":"doi:10.1145/3770761.3777018","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770761.3777018","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.2","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3770761.3777018","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040587523","display_name":"Anant Gupta","orcid":"https://orcid.org/0000-0001-9674-3789"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Anant Gupta","raw_affiliation_strings":["College of Computing, Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"College of Computing, Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059961510","display_name":"H. Nguyen","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hieu Nguyen","raw_affiliation_strings":["College of Computing, Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"College of Computing, Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116061574","display_name":"Carine Geltrudes Webber","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carine Geltrudes Webber","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003232902","display_name":"J. R. Stevens","orcid":"https://orcid.org/0000-0002-0816-200X"},"institutions":[{"id":"https://openalex.org/I204465549","display_name":"Washington University in St. Louis","ror":"https://ror.org/01yc7t268","country_code":"US","type":"education","lineage":["https://openalex.org/I204465549"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Justin Stevens","raw_affiliation_strings":["Washington University in St. Louis, Saint Louis, USA"],"affiliations":[{"raw_affiliation_string":"Washington University in St. Louis, Saint Louis, USA","institution_ids":["https://openalex.org/I204465549"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125888916","display_name":"Abrahim Ladha","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abrahim Ladha","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098669235","display_name":"Sanika Ainchwar","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sanika Ainchwar","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111316278","display_name":"Vasisht L. Ganesh","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vijay Ganesh","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, Canada"],"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, Canada","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5040587523"],"corresponding_institution_ids":["https://openalex.org/I130701444"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.76903207,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1721","last_page":"1721"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.7584999799728394,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.7584999799728394,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.03460000082850456,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.01940000057220459,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8478000164031982},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6398000121116638},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6039000153541565},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5752999782562256},{"id":"https://openalex.org/keywords/intelligent-tutoring-system","display_name":"Intelligent tutoring system","score":0.5091999769210815},{"id":"https://openalex.org/keywords/tuple","display_name":"Tuple","score":0.4339999854564667},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.3578000068664551}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8478000164031982},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7979000210762024},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6398000121116638},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6039000153541565},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5752999782562256},{"id":"https://openalex.org/C2779336797","wikidata":"https://www.wikidata.org/wiki/Q3055492","display_name":"Intelligent tutoring system","level":2,"score":0.5091999769210815},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4934000074863434},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.444599986076355},{"id":"https://openalex.org/C118930307","wikidata":"https://www.wikidata.org/wiki/Q600590","display_name":"Tuple","level":2,"score":0.4339999854564667},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3725999891757965},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.35690000653266907},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.30790001153945923},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.30469998717308044},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.28380000591278076},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28200000524520874},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.2687000036239624}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3770761.3777018","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770761.3777018","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.2","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3770761.3777018","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770761.3777018","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.2","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.8137164115905762,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4,104],"transforming":[5],"Intelligent":[6],"Tutoring":[7],"Systems":[8],"(ITS)":[9],"via":[10],"more":[11,17],"natural":[12],"explanations,":[13],"multi-turn":[14],"dialogue,":[15],"and":[16,33,54,63,77,88,109,166,209],"adaptive":[18],"support":[19],"for":[20,67,185,202],"students.":[21],"Yet":[22],"their":[23],"effectiveness":[24],"depends":[25],"on":[26,39],"rigorous":[27,207],"benchmarking":[28,37],"to":[29,83,107,140,173,193,198],"ensure":[30],"reliability,":[31],"fairness,":[32],"pedagogical":[34],"soundness.":[35],"Such":[36,72],"relies":[38],"detailed":[40],"student":[41,86],"data,":[42],"especially":[43],"data":[44,103,122],"that":[45,154,205],"accurately":[46],"reflects":[47],"the":[48,68,145,156,188,200,217],"actual":[49],"distribution":[50,180],"of":[51,59,79,134,147,151,158,181,212,222],"wrong":[52,61],"answers":[53,62,153],"misconceptions.":[55],"A":[56],"robust":[57],"dataset":[58,74,176],"domain-specific":[60],"misconceptions":[64],"is":[65,197],"critical":[66],"ITS":[69,81],"research":[70],"community.":[71],"a":[73,120,132,149,175,178],"enables":[75],"training":[76],"testing":[78],"LLM-based":[80,213],"designed":[82],"correct":[84],"misconceived":[85],"responses":[87],"guide":[89],"students":[90,160],"appropriately.":[91],"Unfortunately,":[92],"in":[93,126,216],"advanced":[94],"areas":[95,221],"such":[96,102],"as":[97],"Theoretical":[98,227],"Computer":[99,228],"Science":[100],"(TCS),":[101],"scarce,":[105],"costly":[106],"collect,":[108],"limited":[110],"by":[111],"privacy":[112],"concerns.":[113],"To":[114],"address":[115],"this":[116],"problem,":[117],"we":[118,130,170],"propose":[119],"synthetic":[121],"generation":[123],"technique":[124,189],"grounded":[125],"real-world":[127],"data.":[128],"First,":[129],"curate":[131],"set":[133],"human-generated":[135],"(question,":[136],"answer,":[137],"misconception)":[138],"tuples":[139],"seed":[141],"an":[142],"LLM":[143],"with":[144,177],"goal":[146,196],"generating":[148],"corpus":[150],"incorrect":[152],"resemble":[155],"kinds":[157],"mistakes":[159],"make":[161],"while":[162],"solving":[163],"undergraduate-level":[164],"math":[165],"algorithmic":[167],"problems.":[168],"Then,":[169],"prompt":[171],"LLMs":[172],"generate":[174],"similar":[179],"mistakes.":[182],"Once":[183],"validated":[184],"one":[186],"topic,":[187],"can":[190],"be":[191],"transferred":[192],"others.":[194],"Our":[195],"lay":[199],"groundwork":[201],"scalable":[203],"benchmarks":[204],"enable":[206],"evaluation":[208],"broader":[210],"adoption":[211],"tutoring":[214],"systems":[215],"most":[218],"conceptually":[219],"demanding":[220],"computer":[223],"science":[224],"education,":[225],"namely,":[226],"Science.":[229]},"counts_by_year":[],"updated_date":"2026-02-15T05:58:04.055770","created_date":"2026-02-14T00:00:00"}