{"id":"https://openalex.org/W4392858442","doi":"https://doi.org/10.1145/3626253.3635404","title":"The Future of the Error Message: Comparing Large Language Models and Novice Programmer Effectiveness in Fixing Errors","display_name":"The Future of the Error Message: Comparing Large Language Models and Novice Programmer Effectiveness in Fixing Errors","publication_year":2024,"publication_date":"2024-03-14","ids":{"openalex":"https://openalex.org/W4392858442","doi":"https://doi.org/10.1145/3626253.3635404"},"language":"en","primary_location":{"id":"doi:10.1145/3626253.3635404","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626253.3635404","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th ACM Technical Symposium on Computer Science Education V. 2","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5094166309","display_name":"Brij Howard-Sarin","orcid":"https://orcid.org/0009-0005-6720-925X"},"institutions":[{"id":"https://openalex.org/I22675022","display_name":"Williams College","ror":"https://ror.org/04avkmd49","country_code":"US","type":"education","lineage":["https://openalex.org/I22675022"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Brij Howard-Sarin","raw_affiliation_strings":["Computer Science, Williams College, Williamstown, MA, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science, Williams College, Williamstown, MA, USA","institution_ids":["https://openalex.org/I22675022"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5094166309"],"corresponding_institution_ids":["https://openalex.org/I22675022"],"apc_list":null,"apc_paid":null,"fwci":0.7794,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.73243732,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"1881","last_page":"1881"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9772999882698059,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9732000231742859,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/programmer","display_name":"Programmer","score":0.9237416982650757},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8291864395141602},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6288286447525024}],"concepts":[{"id":"https://openalex.org/C2778514511","wikidata":"https://www.wikidata.org/wiki/Q1374194","display_name":"Programmer","level":2,"score":0.9237416982650757},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8291864395141602},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6288286447525024}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626253.3635404","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626253.3635404","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th ACM Technical Symposium on Computer Science Education V. 2","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6600000262260437,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2379153735","https://openalex.org/W2046172023","https://openalex.org/W2972896947","https://openalex.org/W2170146914","https://openalex.org/W2355105570","https://openalex.org/W2083974823","https://openalex.org/W622882839","https://openalex.org/W2185017216","https://openalex.org/W2016345697"],"abstract_inverted_index":{"Research":[0],"on":[1,210,233],"enhancing":[2],"error":[3,58,141,153,177,195],"message":[4,59,142],"presentation":[5],"is":[6],"of":[7,52,66,72,96,108,116,140,151,180],"great":[8],"interest":[9],"to":[10,38,48,82,89,124,193,206,220,229,237],"teachers":[11],"and":[12,25,41,122,126,158,240],"developers":[13],"alike":[14],"because":[15],"improving":[16],"Integrated":[17],"Development":[18],"Environments":[19],"(IDEs)":[20],"increases":[21],"early":[22],"student":[23],"retention":[24],"efficiency":[26],"at":[27,167],"all":[28,70,152],"levels":[29],"with":[30,118],"more":[31],"effective":[32],"developing":[33],"tools.":[34],"This":[35,130,183],"study":[36,131,184],"aims":[37],"compare":[39],"GPT-4":[40,102,157,171,187,215],"novice":[42,78,242],"programmer":[43],"accuracy":[44],"in":[45,197],"fixing":[46],"errors":[47,148],"assess":[49],"the":[50,84,97,128,144,181,222],"viability":[51],"Large":[53],"Language":[54],"Models":[55],"as":[56,190,203,225],"an":[57,75,198],"enhancement":[60],"tool.":[61],"First,":[62],"a":[63,113,137,191,204,226],"random":[64],"sample":[65],"100,000":[67],"sessions":[68],"from":[69,110],"users":[71],"BlueJ":[73],"5,":[74],"IDE":[76],"for":[77,94,176],"programmers,":[79],"was":[80,103,161],"analyzed":[81],"determine":[83],"time":[85,231],"it":[86,160],"took":[87],"programmers":[88],"resolve":[90],"coding":[91],"errors.":[92,129],"Subsequently,":[93],"each":[95],"five":[98,145],"most":[99,146],"common":[100,147],"errors,":[101,235],"given":[104],"20":[105],"randomly-selected":[106],"snippets":[107],"code":[109,120,166,209],"Blackbox":[111,117],"mini,":[112],"curated":[114],"subset":[115],"source":[119],"attached,":[121],"prompted":[123],"explain":[125,194],"fix":[127,165],"replicated":[132],"prior":[133],"research":[134],"that":[135,163,186],"proposed":[136],"Zipf-Mandelbrot":[138],"Distribution":[139],"frequency;":[143],"comprised":[149],"45%":[150],"messages.":[154],"In":[155,213],"comparing":[156],"novices,":[159],"found":[162],"humans":[164],"higher":[168],"rates,":[169],"but":[170],"provided":[172],"completely":[173],"correct":[174,208],"explanations":[175],"messages":[178,196],"96%":[179],"time.":[182],"concludes":[185],"functions":[188],"best":[189,218],"tool":[192,205],"interactive":[199],"format,":[200],"rather":[201],"than":[202],"produce":[207],"its":[211],"own.":[212],"conclusion,":[214],"would":[216],"be":[217],"utilized":[219],"enhance":[221],"classroom":[223],"experience":[224],"chat":[227],"assistant":[228],"reduce":[230],"spent":[232],"syntactical":[234],"leading":[236],"improved":[238],"productivity":[239],"better":[241],"retention.":[243]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}