{"id":"https://openalex.org/W4391584331","doi":"https://doi.org/10.1145/3631802.3631830","title":"CodeHelp: Using Large Language Models with Guardrails for Scalable Support in Programming Classes","display_name":"CodeHelp: Using Large Language Models with Guardrails for Scalable Support in Programming Classes","publication_year":2023,"publication_date":"2023-11-13","ids":{"openalex":"https://openalex.org/W4391584331","doi":"https://doi.org/10.1145/3631802.3631830"},"language":"en","primary_location":{"id":"doi:10.1145/3631802.3631830","doi":"https://doi.org/10.1145/3631802.3631830","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3631802.3631830","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3631802.3631830","source":{"id":"https://openalex.org/S4306420430","display_name":"Koli Calling International Conference on Computing Education Research","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3631802.3631830","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092645205","display_name":"Mark Liffiton","orcid":"https://orcid.org/0009-0004-1512-7829","observed_orcids":["https://orcid.org/0009-0004-1512-7829"]},"institutions":[{"id":"https://openalex.org/I184616310","display_name":"Illinois Wesleyan University","ror":"https://ror.org/021aqk126","country_code":"US","type":"education","lineage":["https://openalex.org/I184616310"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Liffiton","raw_affiliation_strings":["Computer Science, Illinois Wesleyan University, USA"],"raw_orcid":"https://orcid.org/0009-0004-1512-7829","affiliations":[{"raw_affiliation_string":"Computer Science, Illinois Wesleyan University, USA","institution_ids":["https://openalex.org/I184616310"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033414319","display_name":"Brad E. Sheese","orcid":"https://orcid.org/0000-0002-8762-5622","observed_orcids":["https://orcid.org/0000-0002-8762-5622"]},"institutions":[{"id":"https://openalex.org/I184616310","display_name":"Illinois Wesleyan University","ror":"https://ror.org/021aqk126","country_code":"US","type":"education","lineage":["https://openalex.org/I184616310"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brad E Sheese","raw_affiliation_strings":["Computer Science, Illinois Wesleyan University, USA"],"raw_orcid":"https://orcid.org/0000-0002-8762-5622","affiliations":[{"raw_affiliation_string":"Computer Science, Illinois Wesleyan University, USA","institution_ids":["https://openalex.org/I184616310"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077762937","display_name":"Jarom\u00edr \u0160avelka","orcid":"https://orcid.org/0000-0002-3674-5456","observed_orcids":["https://orcid.org/0000-0002-3674-5456"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaromir Savelka","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0002-3674-5456","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032890999","display_name":"Paul C. Denny","orcid":"https://orcid.org/0000-0002-5150-9806","observed_orcids":["https://orcid.org/0000-0002-5150-9806"]},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Paul Denny","raw_affiliation_strings":["School of Computer Science, The University of Auckland, New Zealand"],"raw_orcid":"https://orcid.org/0000-0002-5150-9806","affiliations":[{"raw_affiliation_string":"School of Computer Science, The University of Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":63.6613,"has_fulltext":true,"cited_by_count":285,"citation_normalized_percentile":{"value":0.99860238,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9574999809265137,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9574999809265137,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.02239999920129776,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.009600000455975533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9710000157356262},{"id":"https://openalex.org/keywords/guardrails","display_name":"guardrails","score":0.8740000128746033},{"id":"https://openalex.org/keywords/programming-education","display_name":"programming education","score":0.8270000219345093},{"id":"https://openalex.org/keywords/prompting-strategies","display_name":"prompting strategies","score":0.671999990940094},{"id":"https://openalex.org/keywords/data-science-education","display_name":"data science education","score":0.8209999799728394},{"id":"https://openalex.org/keywords/student-perceptions","display_name":"student perceptions","score":0.6740000247955322},{"id":"https://openalex.org/keywords/instructor-support","display_name":"instructor support","score":0.5799999833106995},{"id":"https://openalex.org/keywords/error-resolution","display_name":"error resolution","score":0.4959999918937683},{"id":"https://openalex.org/keywords/overreliance-on-ai","display_name":"overreliance on AI","score":0.746999979019165}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7983901500701904},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7106478810310364},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5245431065559387},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.4154647886753082},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.32709795236587524},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.13045534491539001}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3631802.3631830","doi":"https://doi.org/10.1145/3631802.3631830","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3631802.3631830","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3631802.3631830","source":{"id":"https://openalex.org/S4306420430","display_name":"Koli Calling International Conference on Computing Education Research","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2308.06921","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.06921","pdf_url":"https://arxiv.org/pdf/2308.06921","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2308.06921","doi":"https://doi.org/10.48550/arxiv.2308.06921","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.06921","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3631802.3631830","doi":"https://doi.org/10.1145/3631802.3631830","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3631802.3631830","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3631802.3631830","source":{"id":"https://openalex.org/S4306420430","display_name":"Koli Calling International Conference on Computing Education Research","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.9901999831199646,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391584331.pdf","grobid_xml":"https://content.openalex.org/works/W4391584331.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W1522875709","https://openalex.org/W1979290264","https://openalex.org/W2783082347","https://openalex.org/W2894540915","https://openalex.org/W2970680470","https://openalex.org/W3177813494","https://openalex.org/W4210521864","https://openalex.org/W4211263275","https://openalex.org/W4213009768","https://openalex.org/W4283170666","https://openalex.org/W4283705032","https://openalex.org/W4286750487","https://openalex.org/W4312081491","https://openalex.org/W4312438588","https://openalex.org/W4318719246","https://openalex.org/W4321012208","https://openalex.org/W4321162272","https://openalex.org/W4323033785","https://openalex.org/W4323037544","https://openalex.org/W4323655724","https://openalex.org/W4362580838","https://openalex.org/W4366808937","https://openalex.org/W4367000100","https://openalex.org/W4378676756","https://openalex.org/W4380353816","https://openalex.org/W4381587445","https://openalex.org/W4382654294","https://openalex.org/W4385474073","https://openalex.org/W4386099272","https://openalex.org/W4387129749","https://openalex.org/W4390315357","https://openalex.org/W4390985631","https://openalex.org/W7133224126"],"related_works":["https://openalex.org/W4390490804","https://openalex.org/W4396833177","https://openalex.org/W7204639567","https://openalex.org/W7147353194","https://openalex.org/W7160562397","https://openalex.org/W4404316579","https://openalex.org/W4391124688","https://openalex.org/W7128814260","https://openalex.org/W4380353816","https://openalex.org/W7127740216"],"abstract_inverted_index":{"Computing":[0],"educators":[1],"face":[2],"significant":[3],"challenges":[4],"in":[5,12,114],"providing":[6,28],"timely":[7],"support":[8,193],"to":[9,63,67,100,185,197],"students,":[10],"especially":[11,169],"large":[13,33],"class":[14],"settings.":[15],"Large":[16],"language":[17],"models":[18],"(LLMs)":[19],"have":[20],"emerged":[21],"recently":[22],"and":[23,90,118,125,138,143,151,173,178,187],"show":[24],"great":[25],"promise":[26],"for":[27,88,106,156,180],"on-demand":[29,65],"help":[30,174],"at":[31],"a":[32,56,83,115,130,152],"scale,":[34],"but":[35],"there":[36],"are":[37,104],"concerns":[38],"that":[39,162,179,194],"students":[40,69,124,167],"may":[41],"over-rely":[42],"on":[43,92],"the":[44,76,79,93,141,148,192],"outputs":[45,103],"produced":[46],"by":[47,166],"these":[48],"models.":[49],"In":[50],"this":[51],"paper,":[52],"we":[53,98,111,144],"introduce":[54],"CodeHelp,":[55,110],"novel":[57],"LLM-powered":[58],"tool":[59],"designed":[60],"with":[61,122,175],"guardrails":[62],"provide":[64,196],"assistance":[66],"programming":[68],"without":[70],"directly":[71],"revealing":[72],"solutions.":[73],"We":[74,133],"detail":[75],"design":[77],"of":[78,85,95,140,154],"tool,":[80,142],"which":[81],"incorporates":[82],"number":[84],"useful":[86],"features":[87],"instructors,":[89],"elaborate":[91],"pipeline":[94],"prompting":[96],"strategies":[97],"use":[99],"ensure":[101],"generated":[102],"suitable":[105],"students.":[107,198],"To":[108],"evaluate":[109],"deployed":[112],"it":[113,182],"first-year":[116],"computer":[117],"data":[119],"science":[120],"course":[121,149],"52":[123],"collected":[126],"student":[127],"interactions":[128],"over":[129],"12-week":[131],"period.":[132],"examine":[134],"students\u2019":[135],"usage":[136],"patterns":[137],"perceptions":[139],"report":[145],"reflections":[146],"from":[147],"instructor":[150],"series":[153],"recommendations":[155],"classroom":[157],"use.":[158],"Our":[159],"findings":[160],"suggest":[161],"CodeHelp":[163],"is":[164,183],"well-received":[165],"who":[168],"value":[170],"its":[171],"availability":[172],"resolving":[176],"errors,":[177],"instructors":[181],"easy":[184],"deploy":[186],"complements,":[188],"rather":[189],"than":[190],"replaces,":[191],"they":[195]},"counts_by_year":[{"year":2026,"cited_by_count":97},{"year":2025,"cited_by_count":106},{"year":2024,"cited_by_count":78},{"year":2023,"cited_by_count":3},{"year":2017,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-02-07T00:00:00"}