{"id":"https://openalex.org/W4392564649","doi":"https://doi.org/10.1145/3626252.3630764","title":"A Large Scale RCT on Effective Error Messages in CS1","display_name":"A Large Scale RCT on Effective Error Messages in CS1","publication_year":2024,"publication_date":"2024-03-07","ids":{"openalex":"https://openalex.org/W4392564649","doi":"https://doi.org/10.1145/3626252.3630764"},"language":"en","primary_location":{"id":"doi:10.1145/3626252.3630764","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626252.3630764","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th ACM Technical Symposium on Computer Science Education V. 1","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017497886","display_name":"Sierra Wang","orcid":"https://orcid.org/0000-0003-1376-8759"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sierra Wang","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101897622","display_name":"John C. Mitchell","orcid":"https://orcid.org/0000-0002-0024-860X"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Mitchell","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074969309","display_name":"Chris Piech","orcid":"https://orcid.org/0000-0001-5140-0467"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Piech","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5017497886"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":11.9414,"has_fulltext":false,"cited_by_count":30,"citation_normalized_percentile":{"value":0.98462094,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1395","last_page":"1401"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9391999840736389,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/demographics","display_name":"Demographics","score":0.7366844415664673},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.681829571723938},{"id":"https://openalex.org/keywords/randomized-experiment","display_name":"Randomized experiment","score":0.566859781742096},{"id":"https://openalex.org/keywords/error-detection-and-correction","display_name":"Error detection and correction","score":0.5416207313537598},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5372064113616943},{"id":"https://openalex.org/keywords/randomized-controlled-trial","display_name":"Randomized controlled trial","score":0.5155129432678223},{"id":"https://openalex.org/keywords/type-i-and-type-ii-errors","display_name":"Type I and type II errors","score":0.5141029953956604},{"id":"https://openalex.org/keywords/intervention","display_name":"Intervention (counseling)","score":0.4726473391056061},{"id":"https://openalex.org/keywords/error-analysis","display_name":"Error analysis","score":0.4577038288116455},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.25399526953697205},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.20902106165885925},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.12071153521537781},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11195299029350281},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.09493079781532288}],"concepts":[{"id":"https://openalex.org/C2780084366","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demographics","level":2,"score":0.7366844415664673},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.681829571723938},{"id":"https://openalex.org/C155108698","wikidata":"https://www.wikidata.org/wiki/Q1231081","display_name":"Randomized experiment","level":2,"score":0.566859781742096},{"id":"https://openalex.org/C103088060","wikidata":"https://www.wikidata.org/wiki/Q1062839","display_name":"Error detection and correction","level":2,"score":0.5416207313537598},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5372064113616943},{"id":"https://openalex.org/C168563851","wikidata":"https://www.wikidata.org/wiki/Q1436668","display_name":"Randomized controlled trial","level":2,"score":0.5155129432678223},{"id":"https://openalex.org/C40696583","wikidata":"https://www.wikidata.org/wiki/Q989120","display_name":"Type I and type II errors","level":2,"score":0.5141029953956604},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.4726473391056061},{"id":"https://openalex.org/C3018824978","wikidata":"https://www.wikidata.org/wiki/Q2894891","display_name":"Error analysis","level":2,"score":0.4577038288116455},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.25399526953697205},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.20902106165885925},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.12071153521537781},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11195299029350281},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.09493079781532288},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.0},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626252.3630764","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626252.3630764","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th ACM Technical Symposium on Computer Science Education V. 1","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.4399999976158142,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W1971916043","https://openalex.org/W1975487129","https://openalex.org/W2050390685","https://openalex.org/W2111760158","https://openalex.org/W2161052636","https://openalex.org/W2296095100","https://openalex.org/W2469269292","https://openalex.org/W2523566863","https://openalex.org/W2604953184","https://openalex.org/W2617813225","https://openalex.org/W2747427841","https://openalex.org/W2791302163","https://openalex.org/W2884648946","https://openalex.org/W2899324569","https://openalex.org/W2915219867","https://openalex.org/W2916390547","https://openalex.org/W2943692676","https://openalex.org/W2963074569","https://openalex.org/W2981301999","https://openalex.org/W2993999284","https://openalex.org/W3000240292","https://openalex.org/W3008941975","https://openalex.org/W3033595121","https://openalex.org/W3086619362","https://openalex.org/W3135705253","https://openalex.org/W3163358064","https://openalex.org/W4323033692","https://openalex.org/W4323033785","https://openalex.org/W4323037584","https://openalex.org/W4382654255","https://openalex.org/W4392542352"],"related_works":["https://openalex.org/W3121380072","https://openalex.org/W2058403539","https://openalex.org/W2333615638","https://openalex.org/W2602311653","https://openalex.org/W2964230772","https://openalex.org/W1218323607","https://openalex.org/W2380243770","https://openalex.org/W2377495875","https://openalex.org/W2390973415","https://openalex.org/W2074837817"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,83],"evaluate":[4,85],"the":[5,65,81,86,113,145,159,182],"most":[6,183],"effective":[7,192],"error":[8,35,50,61,75,94,104,108,119,130,149,164,179,185],"message":[9,36,95,150,186],"types":[10],"through":[11],"a":[12,190],"large-scale":[13],"randomized":[14],"controlled":[15],"trial":[16],"conducted":[17],"in":[18,55,112,120,144,158],"an":[19,107,118,135],"open-access,":[20],"online":[21],"introductory":[22],"computer":[23],"science":[24],"course":[25,66],"with":[26],"8,762":[27],"students":[28,100,127,167],"from":[29,168],"146":[30],"countries.":[31],"We":[32,97,132],"assess":[33],"existing":[34],"enhancement":[37],"strategies,":[38],"as":[39,41,181,189],"well":[40],"two":[42],"novel":[43],"approaches":[44],"of":[45,92,147,161],"our":[46],"own:":[47],"(1)":[48],"generating":[49],"messages":[51,62,105,165,180],"using":[52,101,128],"OpenAI's":[53],"GPT":[54,102,162,177],"real":[56],"time":[57],"and":[58,77,88,116,171],"(2)":[59],"constructing":[60],"that":[63,99],"incorporate":[64],"discussion":[67],"forum.":[68],"By":[69],"examining":[70],"students'":[71],"direct":[72],"responses":[73],"to":[74,126,140],"messages,":[76],"their":[78],"behavior":[79],"throughout":[80],"course,":[82],"quantitatively":[84],"immediate":[87],"longer":[89],"term":[90],"efficacy":[91],"different":[93,148],"types.":[96,151],"find":[98,154],"generated":[103,163,178],"repeat":[106],"23.1%":[109],"less":[110],"often":[111],"subsequent":[114],"attempt,":[115],"resolve":[117],"34.8%":[121],"fewer":[122],"additional":[123],"attempts,":[124],"compared":[125],"standard":[129],"messages.":[131],"also":[133],"perform":[134],"analysis":[136],"across":[137,194],"various":[138],"demographics":[139],"understand":[141],"any":[142],"disparities":[143],"impact":[146],"Our":[152,174],"results":[153],"no":[155],"significant":[156],"difference":[157],"effectiveness":[160],"for":[166],"varying":[169],"socioeconomic":[170],"demographic":[172],"backgrounds.":[173],"findings":[175],"underscore":[176],"helpful":[184],"type,":[187],"especially":[188],"universally":[191],"intervention":[193],"demographics.":[195]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":8}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}