{"id":"https://openalex.org/W4381587445","doi":"https://doi.org/10.1145/3568813.3600142","title":"Thrilled by Your Progress! Large Language Models (GPT-4) No Longer Struggle to Pass Assessments in Higher Education Programming Courses","display_name":"Thrilled by Your Progress! Large Language Models (GPT-4) No Longer Struggle to Pass Assessments in Higher Education Programming Courses","alternate_titles":[],"publication_year":2023,"publication_date":"2023-08-07","ids":{"openalex":"https://openalex.org/W4381587445","doi":"https://doi.org/10.1145/3568813.3600142"},"language":"en","primary_location":{"id":"doi:10.1145/3568813.3600142","doi":"https://doi.org/10.1145/3568813.3600142","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3568813.3600142","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3568813.3600142","source":{"id":"https://openalex.org/S7407085948","display_name":"ACM Conference on International Computing Education Research (ICER)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Conference on International Computing Education Research V.1","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4381587445"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3568813.3600142","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077762937","display_name":"Jarom\u00edr \u0160avelka","orcid":"https://orcid.org/0000-0002-3674-5456","observed_orcids":["https://orcid.org/0000-0002-3674-5456"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaromir Savelka","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0002-3674-5456","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017723217","display_name":"Arav Agarwal","orcid":"https://orcid.org/0000-0001-9848-1663","observed_orcids":["https://orcid.org/0000-0001-9848-1663","https://orcid.org/0009-0001-2507-8387"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arav Agarwal","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0001-9848-1663","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076326571","display_name":"Marshall An","orcid":"https://orcid.org/0009-0005-5165-640X","observed_orcids":["https://orcid.org/0009-0005-5165-640X"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marshall An","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0009-0005-5165-640X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026486394","display_name":"Christopher Bogart","orcid":"https://orcid.org/0000-0001-8581-115X","observed_orcids":["https://orcid.org/0000-0001-8581-115X"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Bogart","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0001-8581-115X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001245246","display_name":"Majd F. Sakr","orcid":"https://orcid.org/0000-0001-5150-8259","observed_orcids":["https://orcid.org/0000-0001-5150-8259"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Majd Sakr","raw_affiliation_strings":["Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0001-5150-8259","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":38.6847,"has_fulltext":true,"cited_by_count":204,"citation_normalized_percentile":{"value":0.99674191,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"78","last_page":"92"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.8823000192642212,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.8823000192642212,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.08980000019073486,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.014000000432133675,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9300000071525574},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":1.0},{"id":"https://openalex.org/keywords/programming-education","display_name":"programming education","score":0.6430000066757202},{"id":"https://openalex.org/keywords/python-programming-course","display_name":"Python programming course","score":0.6570000052452087},{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.7599999904632568},{"id":"https://openalex.org/keywords/automated-grading","display_name":"automated grading","score":0.6179999709129333},{"id":"https://openalex.org/keywords/multiple-choice-questions","display_name":"multiple-choice questions","score":0.8360000252723694},{"id":"https://openalex.org/keywords/programming-exercises","display_name":"programming exercises","score":0.7739999890327454},{"id":"https://openalex.org/keywords/generative-ai-in-higher-education","display_name":"generative AI in higher education","score":0.7960000038146973},{"id":"https://openalex.org/keywords/assessment-design","display_name":"assessment design","score":0.6639999747276306},{"id":"https://openalex.org/keywords/gpt-3","display_name":"GPT-3","score":0.6819999814033508}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6921341419219971},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.666694700717926},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.582578718662262},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.490002304315567},{"id":"https://openalex.org/C2986567400","wikidata":"https://www.wikidata.org/wiki/Q15777","display_name":"C programming language","level":3,"score":0.47506043314933777},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.45441967248916626},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34736716747283936},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3263383209705353},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3224591016769409},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.20530742406845093},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3568813.3600142","doi":"https://doi.org/10.1145/3568813.3600142","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3568813.3600142","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3568813.3600142","source":{"id":"https://openalex.org/S7407085948","display_name":"ACM Conference on International Computing Education Research (ICER)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Conference on International Computing Education Research V.1","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2306.10073","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.10073","pdf_url":"https://arxiv.org/pdf/2306.10073","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2306.10073","doi":"https://doi.org/10.48550/arxiv.2306.10073","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2306.10073","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.1145/3568813.3600142","doi":"https://doi.org/10.1145/3568813.3600142","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3568813.3600142","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3568813.3600142","source":{"id":"https://openalex.org/S7407085948","display_name":"ACM Conference on International Computing Education Research (ICER)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Conference on International Computing Education Research V.1","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality education","score":0.975600004196167,"id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4381587445.pdf","grobid_xml":"https://content.openalex.org/works/W4381587445.grobid-xml"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W2342082360","https://openalex.org/W2466175319","https://openalex.org/W2492304725","https://openalex.org/W2606964149","https://openalex.org/W2626778328","https://openalex.org/W2890894339","https://openalex.org/W3030163527","https://openalex.org/W3083410900","https://openalex.org/W3130319171","https://openalex.org/W3132736064","https://openalex.org/W3173777717","https://openalex.org/W3177813494","https://openalex.org/W3211492005","https://openalex.org/W4211263275","https://openalex.org/W4226278401","https://openalex.org/W4226287673","https://openalex.org/W4281483047","https://openalex.org/W4283705032","https://openalex.org/W4307309259","https://openalex.org/W4310881984","https://openalex.org/W4312438588","https://openalex.org/W4313384047","https://openalex.org/W4315606663","https://openalex.org/W4317502110","https://openalex.org/W4317547647","https://openalex.org/W4319460874","https://openalex.org/W4319662928","https://openalex.org/W4320167623","https://openalex.org/W4321162272","https://openalex.org/W4321177655","https://openalex.org/W4323033692","https://openalex.org/W4323033785","https://openalex.org/W4323033814","https://openalex.org/W4323037544","https://openalex.org/W4327810158","https://openalex.org/W4360836968","https://openalex.org/W4366808937","https://openalex.org/W4377009756","https://openalex.org/W4378676756","https://openalex.org/W4382653968","https://openalex.org/W4382654294","https://openalex.org/W4386099272","https://openalex.org/W4387129749","https://openalex.org/W4387869031","https://openalex.org/W4388858772","https://openalex.org/W4389988584","https://openalex.org/W4392359953","https://openalex.org/W7133223072","https://openalex.org/W7133224126"],"related_works":["https://openalex.org/W4382653968","https://openalex.org/W4391826221","https://openalex.org/W4386044264","https://openalex.org/W4392617381","https://openalex.org/W4391031064","https://openalex.org/W4382766666","https://openalex.org/W4366808937","https://openalex.org/W4394682124","https://openalex.org/W7000679860","https://openalex.org/W7223481966"],"abstract_inverted_index":{"This":[0,102,303],"paper":[1],"studies":[2,52],"recent":[3,243,300],"developments":[4],"in":[5,14,30,46,68,90,108,229,261,316],"large":[6],"language":[7],"models\u2019":[8],"(LLM)":[9],"abilities":[10],"to":[11,78,94,129,150,176,189,214,252,275,285,297,311,331],"pass":[12,79],"assessments":[13,96,141,168,210,281],"introductory":[15],"and":[16,234,273,347],"intermediate":[17],"Python":[18,138],"programming":[19,47,70,152,208,264,280,292,308,345],"courses":[20,139,218],"at":[21],"the":[22,56,72,80,91,105,109,123,130,167,178,182,199,206,217,237,242,277,287,299],"postsecondary":[23],"level.":[24],"The":[25,82],"emergence":[26],"of":[27,33,64,84,111,125,133,181,232,239,245,257,279],"ChatGPT":[28],"resulted":[29],"heated":[31],"debates":[32],"its":[34,187],"potential":[35,251],"uses":[36],"(e.g.,":[37,49],"exercise":[38],"generation,":[39],"code":[40,148,155],"explanation)":[41],"as":[42,44,184,186,282,284,343],"well":[43,60,173,185,283],"misuses":[45],"classes":[48,71,293],"cheating).":[50],"Recent":[51],"show":[53],"that":[54,169,198,307,325],"while":[55],"technology":[57,324],"performs":[58],"surprisingly":[59],"on":[61,136,339],"diverse":[62],"sets":[63],"assessment":[65,258],"instruments":[66],"employed":[67],"typical":[69,207],"performance":[73,124],"is":[74,104,319],"usually":[75],"not":[76,171],"sufficient":[77],"courses.":[81,265],"release":[83],"GPT-4":[85,175],"largely":[86],"emphasized":[87],"notable":[88],"improvements":[89],"capabilities":[92,188],"related":[93],"handling":[95,231],"originally":[97],"designed":[98],"for":[99,313],"human":[100,221],"test-takers.":[101],"study":[103,304],"necessary":[106,288],"analysis":[107],"context":[110],"this":[112],"ongoing":[113],"transition":[114],"towards":[115],"mature":[116],"generative":[117],"AI":[118],"systems.":[119],"Specifically,":[120],"we":[121,165,225],"report":[122],"GPT-4,":[126],"comparing":[127],"it":[128],"previous":[131],"generations":[132,244],"GPT":[134,200,246],"models,":[135],"three":[137],"with":[140,154,219,335],"ranging":[142],"from":[143,203],"simple":[144],"multiple-choice":[145],"questions":[146],"(no":[147],"involved)":[149],"complex":[151],"projects":[153],"bases":[156],"distributed":[157],"into":[158,290],"multiple":[159],"files":[160],"(599":[161],"exercises":[162],"overall).":[163],"Additionally,":[164],"analyze":[166],"were":[170],"handled":[172],"by":[174,193,271,329],"understand":[177],"current":[179],"limitations":[180,228],"model,":[183],"leverage":[190],"feedback":[191],"provided":[192],"an":[194,320],"auto-grader.":[195],"We":[196],"found":[197],"models":[201,247],"evolved":[202],"completely":[204],"failing":[205],"class\u2019":[209],"(the":[211],"original":[212],"GPT-3)":[213],"confidently":[215],"passing":[216,333],"no":[220,336],"involvement":[222],"(GPT-4).":[223],"While":[224],"identified":[226],"certain":[227],"GPT-4\u2019s":[230],"MCQs":[233],"coding":[235],"exercises,":[236],"rate":[238],"improvement":[240],"across":[241],"strongly":[248],"suggests":[249],"their":[250],"handle":[253],"almost":[254],"any":[255],"type":[256],"widely":[259,322],"used":[260],"higher":[262],"education":[263],"These":[266],"findings":[267],"could":[268],"be":[269,295,327],"leveraged":[270],"educators":[272],"institutions":[274],"adapt":[276],"design":[278],"fuel":[286],"discussions":[289],"how":[291],"should":[294],"updated":[296],"reflect":[298],"technological":[301],"developments.":[302],"provides":[305],"evidence":[306],"instructors":[309],"need":[310],"prepare":[312],"a":[314],"world":[315],"which":[317],"there":[318],"easy-to-use":[321],"accessible":[323],"can":[326],"utilized":[328],"learners":[330],"collect":[332],"scores,":[334],"effort":[337],"whatsoever,":[338],"what":[340],"today":[341],"counts":[342],"viable":[344],"knowledge":[346],"skills":[348],"assessments.":[349]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4381587445","counts_by_year":[{"year":2026,"cited_by_count":47},{"year":2025,"cited_by_count":70},{"year":2024,"cited_by_count":73},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":1}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2023-06-22T00:00:00"}
