{"id":"https://openalex.org/W4390490882","doi":"https://doi.org/10.1145/3636243.3636245","title":"Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models","display_name":"Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models","publication_year":2024,"publication_date":"2024-01-02","ids":{"openalex":"https://openalex.org/W4390490882","doi":"https://doi.org/10.1145/3636243.3636245"},"language":"en","primary_location":{"id":"doi:10.1145/3636243.3636245","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3636243.3636245","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3636243.3636245","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th Australasian Computing Education Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3636243.3636245","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042822346","display_name":"Stephen MacNeil","orcid":"https://orcid.org/0000-0003-2781-6619"},"institutions":[{"id":"https://openalex.org/I2801004183","display_name":"Temple College","ror":"https://ror.org/038s1ax16","country_code":"US","type":"education","lineage":["https://openalex.org/I2801004183"]},{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Stephen Macneil","raw_affiliation_strings":["Temple University, United States"],"raw_orcid":"https://orcid.org/0000-0003-2781-6619","affiliations":[{"raw_affiliation_string":"Temple University, United States","institution_ids":["https://openalex.org/I2801004183","https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032890999","display_name":"Paul Denny","orcid":"https://orcid.org/0000-0002-5150-9806"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Paul Denny","raw_affiliation_strings":["The University of Auckland, New Zealand"],"raw_orcid":"https://orcid.org/0000-0002-5150-9806","affiliations":[{"raw_affiliation_string":"The University of Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074358617","display_name":"Andrew Tran","orcid":"https://orcid.org/0000-0002-0094-1113"},"institutions":[{"id":"https://openalex.org/I2801004183","display_name":"Temple College","ror":"https://ror.org/038s1ax16","country_code":"US","type":"education","lineage":["https://openalex.org/I2801004183"]},{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Tran","raw_affiliation_strings":["Temple University, United States"],"raw_orcid":"https://orcid.org/0000-0002-0094-1113","affiliations":[{"raw_affiliation_string":"Temple University, United States","institution_ids":["https://openalex.org/I2801004183","https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041367899","display_name":"Juho Leinonen","orcid":"https://orcid.org/0000-0001-6829-9449"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Juho Leinonen","raw_affiliation_strings":["The University of Auckland, New Zealand"],"raw_orcid":"https://orcid.org/0000-0001-6829-9449","affiliations":[{"raw_affiliation_string":"The University of Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Seth Bernstein","orcid":"https://orcid.org/0000-0001-5767-1057"},"institutions":[{"id":"https://openalex.org/I2801004183","display_name":"Temple College","ror":"https://ror.org/038s1ax16","country_code":"US","type":"education","lineage":["https://openalex.org/I2801004183"]},{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Seth Bernstein","raw_affiliation_strings":["Temple University, United States"],"raw_orcid":"https://orcid.org/0000-0001-5767-1057","affiliations":[{"raw_affiliation_string":"Temple University, United States","institution_ids":["https://openalex.org/I2801004183","https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076828114","display_name":"Arto Hellas","orcid":"https://orcid.org/0000-0001-6502-209X"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Arto Hellas","raw_affiliation_strings":["Aalto University, Finland"],"raw_orcid":"https://orcid.org/0000-0001-6502-209X","affiliations":[{"raw_affiliation_string":"Aalto University, Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064114707","display_name":"Sami Sarsa","orcid":"https://orcid.org/0000-0002-7277-9282"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Sami Sarsa","raw_affiliation_strings":["Aalto University, Finland"],"raw_orcid":"https://orcid.org/0000-0002-7277-9282","affiliations":[{"raw_affiliation_string":"Aalto University, Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101999351","display_name":"Joanne Kim","orcid":"https://orcid.org/0000-0001-7646-2373"},"institutions":[{"id":"https://openalex.org/I2801004183","display_name":"Temple College","ror":"https://ror.org/038s1ax16","country_code":"US","type":"education","lineage":["https://openalex.org/I2801004183"]},{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joanne Kim","raw_affiliation_strings":["Temple University, United States"],"raw_orcid":"https://orcid.org/0000-0001-7646-2373","affiliations":[{"raw_affiliation_string":"Temple University, United States","institution_ids":["https://openalex.org/I2801004183","https://openalex.org/I84392919"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5042822346"],"corresponding_institution_ids":["https://openalex.org/I2801004183","https://openalex.org/I84392919"],"apc_list":null,"apc_paid":null,"fwci":12.0682,"has_fulltext":true,"cited_by_count":26,"citation_normalized_percentile":{"value":0.98819876,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"11","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7829732894897461},{"id":"https://openalex.org/keywords/compiler","display_name":"Compiler","score":0.6365188360214233},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.49739935994148254},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4940791428089142},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.4850153625011444},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4624265730381012},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4479481875896454},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.430019736289978},{"id":"https://openalex.org/keywords/interpreter","display_name":"Interpreter","score":0.420210599899292},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.32348737120628357},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31020912528038025},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.17495816946029663}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7829732894897461},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.6365188360214233},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.49739935994148254},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4940791428089142},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.4850153625011444},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4624265730381012},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4479481875896454},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.430019736289978},{"id":"https://openalex.org/C122783720","wikidata":"https://www.wikidata.org/wiki/Q183065","display_name":"Interpreter","level":2,"score":0.420210599899292},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32348737120628357},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31020912528038025},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.17495816946029663},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3636243.3636245","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3636243.3636245","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3636243.3636245","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th Australasian Computing Education Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:aaltodoc.aalto.fi:123456789/127467","is_oa":true,"landing_page_url":"https://research.aalto.fi/en/publications/d7eeefb6-b689-4c60-a00d-db8004c1f440","pdf_url":null,"source":{"id":"https://openalex.org/S4306401663","display_name":"Aaltodoc (Aalto University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I9927081","host_organization_name":"Aalto University","host_organization_lineage":["https://openalex.org/I9927081"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"publishedVersion"}],"best_oa_location":{"id":"doi:10.1145/3636243.3636245","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3636243.3636245","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3636243.3636245","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th Australasian Computing Education Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8899999856948853}],"awards":[],"funders":[{"id":"https://openalex.org/F4320322862","display_name":"Ulla Tuomisen S\u00e4\u00e4ti\u00f6","ror":"https://ror.org/0066s0r72"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4390490882.pdf","grobid_xml":"https://content.openalex.org/works/W4390490882.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W133284495","https://openalex.org/W1971916043","https://openalex.org/W2004451884","https://openalex.org/W2009908982","https://openalex.org/W2013831574","https://openalex.org/W2051669404","https://openalex.org/W2058580210","https://openalex.org/W2076771354","https://openalex.org/W2122166962","https://openalex.org/W2281037989","https://openalex.org/W2395964354","https://openalex.org/W2593433351","https://openalex.org/W2612252595","https://openalex.org/W2783796210","https://openalex.org/W2915219867","https://openalex.org/W2916390547","https://openalex.org/W2962230923","https://openalex.org/W3000240292","https://openalex.org/W3007422522","https://openalex.org/W3030391634","https://openalex.org/W3133959936","https://openalex.org/W3163358064","https://openalex.org/W4211263275","https://openalex.org/W4283705032","https://openalex.org/W4311426581","https://openalex.org/W4323033692","https://openalex.org/W4323033814","https://openalex.org/W4323037544","https://openalex.org/W4323526518","https://openalex.org/W4324138978","https://openalex.org/W4366808937","https://openalex.org/W4381587445","https://openalex.org/W4382515993","https://openalex.org/W4382654294","https://openalex.org/W4384345745","https://openalex.org/W4385932664","https://openalex.org/W4386584937","https://openalex.org/W4386707723","https://openalex.org/W4388584565","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W1721418433","https://openalex.org/W2463512549","https://openalex.org/W2246915267","https://openalex.org/W2004305899","https://openalex.org/W1844878253","https://openalex.org/W2611574984","https://openalex.org/W3089412887","https://openalex.org/W2548985027","https://openalex.org/W2412406728","https://openalex.org/W4380149490"],"abstract_inverted_index":{"Identifying":[0],"and":[1,71,117,151,175,179,214,228,233,254],"resolving":[2],"logic":[3,29,90,142,185,222],"errors":[4,30,61,143],"can":[5,25,31,62,73],"be":[6,32,51,63,77,155,248],"one":[7],"of":[8,112,136,149,170,184,195,212,231],"the":[9,48,69,131,139,146,168,203,226],"most":[10],"frustrating":[11],"challenges":[12],"for":[13,19,98,109,177,258],"novices":[14],"programmers.":[15],"Unlike":[16],"syntax":[17],"errors,":[18,91],"which":[20,128],"a":[21,27,54,110,181,192,209],"compiler":[22],"or":[23],"interpreter":[24],"issue":[26,49],"message,":[28],"subtle.":[33],"In":[34],"certain":[35],"conditions,":[36],"buggy":[37],"code":[38,126,150],"may":[39,153],"even":[40],"exhibit":[41],"correct":[42],"behavior":[43,135],"\u2013":[44],"in":[45,87,221],"other":[46,140],"cases,":[47],"might":[50],"about":[52],"how":[53,244],"problem":[55],"statement":[56],"has":[57],"been":[58],"interpreted.":[59],"Such":[60],"hard":[64],"to":[65,125,145,159],"spot":[66],"when":[67,93,261],"reading":[68],"code,":[70],"they":[72],"also":[74],"at":[75],"times":[76],"missed":[78],"by":[79,161],"automated":[80],"tests.":[81],"There":[82],"is":[83],"great":[84],"educational":[85],"potential":[86,257],"automatically":[88],"detecting":[89,178],"especially":[92],"paired":[94],"with":[95,130,191],"suitable":[96],"feedback":[97],"novices.":[99],"Large":[100],"language":[101],"models":[102,246],"(LLMs)":[103],"have":[104],"recently":[105],"demonstrated":[106],"surprising":[107],"performance":[108,148,169,190],"range":[111],"computing":[113,197,251],"tasks,":[114],"including":[115],"generating":[116],"explaining":[118],"code.":[119],"These":[120],"capabilities":[121],"are":[122],"closely":[123],"linked":[124],"syntax,":[127],"aligns":[129],"next":[132],"token":[133],"prediction":[134],"LLMs.":[137,162],"On":[138],"hand,":[141],"relate":[144],"runtime":[147],"thus":[152],"not":[154],"as":[156],"well":[157],"suited":[158],"analysis":[160,211],"To":[163],"explore":[164],"this,":[165],"we":[166,217],"investigate":[167],"two":[171],"popular":[172],"LLMs,":[173,232],"GPT-3":[174],"GPT-4,":[176],"providing":[180],"novice-friendly":[182],"explanation":[183],"errors.":[186],"We":[187,242],"compare":[188],"LLM":[189,237],"large":[193],"cohort":[194],"introductory":[196],"students":[198,260],"(n":[199],"=":[200],"964)":[201],"solving":[202],"same":[204],"error":[205,223],"detection":[206],"task.":[207],"Through":[208],"mixed-methods":[210],"student":[213],"model":[215],"responses,":[216],"observe":[218],"significant":[219],"improvement":[220],"identification":[224],"between":[225],"previous":[227],"current":[229],"generation":[230],"find":[234],"that":[235],"both":[236],"generations":[238],"significantly":[239],"outperform":[240],"students.":[241],"outline":[243],"such":[245],"could":[247],"integrated":[249],"into":[250],"education":[252],"tools,":[253],"discuss":[255],"their":[256],"supporting":[259],"learning":[262],"programming.":[263]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":10}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}