{"id":"https://openalex.org/W3163358064","doi":"https://doi.org/10.1145/3411764.3445696","title":"On Designing Programming Error Messages for Novices: Readability and its Constituent Factors","display_name":"On Designing Programming Error Messages for Novices: Readability and its Constituent Factors","publication_year":2021,"publication_date":"2021-05-06","ids":{"openalex":"https://openalex.org/W3163358064","doi":"https://doi.org/10.1145/3411764.3445696","mag":"3163358064"},"language":"en","primary_location":{"id":"doi:10.1145/3411764.3445696","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445696","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445696","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445696","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032890999","display_name":"Paul Denny","orcid":"https://orcid.org/0000-0002-5150-9806"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":true,"raw_author_name":"Paul Denny","raw_affiliation_strings":["The University of Auckland, New Zealand"],"affiliations":[{"raw_affiliation_string":"The University of Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065053503","display_name":"James Prather","orcid":"https://orcid.org/0000-0003-2807-6042"},"institutions":[{"id":"https://openalex.org/I60205797","display_name":"Abilene Christian University","ror":"https://ror.org/004srrf86","country_code":"US","type":"education","lineage":["https://openalex.org/I60205797"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Prather","raw_affiliation_strings":["School of Information Technology and Computing Abilene Christian University, United States"],"affiliations":[{"raw_affiliation_string":"School of Information Technology and Computing Abilene Christian University, United States","institution_ids":["https://openalex.org/I60205797"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054692443","display_name":"Brett A. Becker","orcid":"https://orcid.org/0000-0003-1446-647X"},"institutions":[{"id":"https://openalex.org/I100930933","display_name":"University College Dublin","ror":"https://ror.org/05m7pjf47","country_code":"IE","type":"education","lineage":["https://openalex.org/I100930933"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Brett A. Becker","raw_affiliation_strings":["School of Computer Science University College Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"School of Computer Science University College Dublin, Ireland","institution_ids":["https://openalex.org/I100930933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061456583","display_name":"Catherine Mooney","orcid":"https://orcid.org/0000-0002-7696-1364"},"institutions":[{"id":"https://openalex.org/I100930933","display_name":"University College Dublin","ror":"https://ror.org/05m7pjf47","country_code":"IE","type":"education","lineage":["https://openalex.org/I100930933"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Catherine Mooney","raw_affiliation_strings":["School of Computer Science University College Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"School of Computer Science University College Dublin, Ireland","institution_ids":["https://openalex.org/I100930933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079211489","display_name":"John Homer","orcid":"https://orcid.org/0009-0002-3454-9600"},"institutions":[{"id":"https://openalex.org/I60205797","display_name":"Abilene Christian University","ror":"https://ror.org/004srrf86","country_code":"US","type":"education","lineage":["https://openalex.org/I60205797"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Homer","raw_affiliation_strings":["Abilene Christian University, United States"],"affiliations":[{"raw_affiliation_string":"Abilene Christian University, United States","institution_ids":["https://openalex.org/I60205797"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022628429","display_name":"Zachary Albrecht","orcid":null},"institutions":[{"id":"https://openalex.org/I60205797","display_name":"Abilene Christian University","ror":"https://ror.org/004srrf86","country_code":"US","type":"education","lineage":["https://openalex.org/I60205797"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zachary C Albrecht","raw_affiliation_strings":["School of Information Technology and Computing Abilene Christian University, United States"],"affiliations":[{"raw_affiliation_string":"School of Information Technology and Computing Abilene Christian University, United States","institution_ids":["https://openalex.org/I60205797"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027028096","display_name":"Garrett Powell","orcid":"https://orcid.org/0000-0002-3221-7015"},"institutions":[{"id":"https://openalex.org/I60205797","display_name":"Abilene Christian University","ror":"https://ror.org/004srrf86","country_code":"US","type":"education","lineage":["https://openalex.org/I60205797"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Garrett B. Powell","raw_affiliation_strings":["Abilene Christian University, United States"],"affiliations":[{"raw_affiliation_string":"Abilene Christian University, United States","institution_ids":["https://openalex.org/I60205797"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5032890999"],"corresponding_institution_ids":["https://openalex.org/I154130895"],"apc_list":null,"apc_paid":null,"fwci":13.4949,"has_fulltext":true,"cited_by_count":60,"citation_normalized_percentile":{"value":0.98788683,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"15"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9890000224113464,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8906213045120239},{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.8803279399871826},{"id":"https://openalex.org/keywords/jargon","display_name":"Jargon","score":0.8277422189712524},{"id":"https://openalex.org/keywords/programmer","display_name":"Programmer","score":0.6528630256652832},{"id":"https://openalex.org/keywords/compiler","display_name":"Compiler","score":0.5573246479034424},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.5170472264289856},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.5168269276618958},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4688115417957306},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4534554183483124},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.44433826208114624},{"id":"https://openalex.org/keywords/word-error-rate","display_name":"Word error rate","score":0.43042320013046265},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.34818917512893677},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.32780930399894714},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2990952134132385}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8906213045120239},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.8803279399871826},{"id":"https://openalex.org/C2777611551","wikidata":"https://www.wikidata.org/wiki/Q17951","display_name":"Jargon","level":2,"score":0.8277422189712524},{"id":"https://openalex.org/C2778514511","wikidata":"https://www.wikidata.org/wiki/Q1374194","display_name":"Programmer","level":2,"score":0.6528630256652832},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.5573246479034424},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.5170472264289856},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.5168269276618958},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4688115417957306},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4534554183483124},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.44433826208114624},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.43042320013046265},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.34818917512893677},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32780930399894714},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2990952134132385},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3411764.3445696","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445696","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445696","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:researchrepository.ucd.ie:10197/24389","is_oa":false,"landing_page_url":"http://hdl.handle.net/10197/24389","pdf_url":null,"source":{"id":"https://openalex.org/S4306402280","display_name":"Research Repository UCD (University College Dublin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I100930933","host_organization_name":"University College Dublin","host_organization_lineage":["https://openalex.org/I100930933"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Conference Publication"}],"best_oa_location":{"id":"doi:10.1145/3411764.3445696","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445696","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445696","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8799999952316284,"display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320320801","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3163358064.pdf","grobid_xml":"https://content.openalex.org/works/W3163358064.grobid-xml"},"referenced_works_count":59,"referenced_works":["https://openalex.org/W58732440","https://openalex.org/W262436420","https://openalex.org/W1530314747","https://openalex.org/W1563185315","https://openalex.org/W1603506326","https://openalex.org/W1620491971","https://openalex.org/W1964413824","https://openalex.org/W1965093370","https://openalex.org/W1965854491","https://openalex.org/W1969792306","https://openalex.org/W1974047325","https://openalex.org/W1981108270","https://openalex.org/W1986204096","https://openalex.org/W1992007857","https://openalex.org/W2009908982","https://openalex.org/W2013935724","https://openalex.org/W2033432579","https://openalex.org/W2047764386","https://openalex.org/W2050390685","https://openalex.org/W2050968492","https://openalex.org/W2069254533","https://openalex.org/W2076949411","https://openalex.org/W2079166885","https://openalex.org/W2082719072","https://openalex.org/W2083433145","https://openalex.org/W2089736546","https://openalex.org/W2146496290","https://openalex.org/W2154948082","https://openalex.org/W2157148654","https://openalex.org/W2161052636","https://openalex.org/W2164996482","https://openalex.org/W2296095100","https://openalex.org/W2521357556","https://openalex.org/W2523566863","https://openalex.org/W2531756803","https://openalex.org/W2604953184","https://openalex.org/W2617813225","https://openalex.org/W2625849667","https://openalex.org/W2747427841","https://openalex.org/W2761740351","https://openalex.org/W2788923553","https://openalex.org/W2791302163","https://openalex.org/W2796264325","https://openalex.org/W2799855324","https://openalex.org/W2884648946","https://openalex.org/W2899324569","https://openalex.org/W2915219867","https://openalex.org/W2917563899","https://openalex.org/W2943692676","https://openalex.org/W2963074569","https://openalex.org/W2965071040","https://openalex.org/W2981301999","https://openalex.org/W2993999284","https://openalex.org/W3000240292","https://openalex.org/W3033595121","https://openalex.org/W3080463073","https://openalex.org/W3138942564","https://openalex.org/W4236130975","https://openalex.org/W4238774439"],"related_works":["https://openalex.org/W1964661231","https://openalex.org/W4254960163","https://openalex.org/W3110264473","https://openalex.org/W2999103397","https://openalex.org/W2370831213","https://openalex.org/W4315704968","https://openalex.org/W4285478380","https://openalex.org/W2598956343","https://openalex.org/W4234691913","https://openalex.org/W1996895614"],"abstract_inverted_index":{"Programming":[0],"error":[1,17,39,77,148],"messages":[2,40,99],"play":[3],"an":[4],"important":[5],"role":[6],"in":[7,90],"learning":[8],"to":[9,62,97,142],"program.":[10],"The":[11],"cycle":[12],"of":[13,53],"program":[14],"input":[15],"and":[16,26,29,36,64,89,123,138],"message":[18,54,78,115,117,136],"response":[19],"completes":[20],"a":[21,31],"loop":[22],"between":[23,34],"the":[24,27,51,81,91],"programmer":[25],"compiler/interpreter":[28],"is":[30,57],"fundamental":[32],"interaction":[33],"human":[35],"computer.":[37],"However,":[38],"are":[41],"notoriously":[42],"problematic,":[43],"especially":[44],"for":[45,130,146],"novices.":[46],"Despite":[47],"numerous":[48],"guidelines":[49,134],"citing":[50],"importance":[52],"readability,":[55],"there":[56],"little":[58],"empirical":[59,128],"research":[60],"dedicated":[61],"understanding":[63],"assessing":[65],"it.":[66],"We":[67,106],"report":[68],"three":[69],"related":[70],"experiments":[71,84],"investigating":[72],"factors":[73,112],"that":[74,109],"infuence":[75],"programming":[76,147],"readability.":[79],"In":[80],"frst":[82],"two":[83],"we":[85,93],"identify":[86],"possible":[87],"factors,":[88],"third":[92],"ask":[94],"novice":[95],"programmers":[96],"rate":[98],"using":[100],"scales":[101],"derived":[102],"from":[103],"these":[104],"factors.":[105],"fnd":[107],"evidence":[108],"several":[110],"key":[111],"signifcantly":[113],"afect":[114],"readability:":[116],"length,":[118],"jargon":[119],"use,":[120],"sentence":[121],"structure,":[122],"vocabulary.":[124],"This":[125],"provides":[126],"novel":[127],"support":[129],"previously":[131],"untested":[132],"longstanding":[133],"on":[135],"design,":[137],"informs":[139],"future":[140],"eforts":[141],"create":[143],"readability":[144],"metrics":[145],"messages.":[149]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":19},{"year":2023,"cited_by_count":20},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}