{"id":"https://openalex.org/W4392617381","doi":"https://doi.org/10.1145/3649217.3653594","title":"Feedback-Generation for Programming Exercises With GPT-4","display_name":"Feedback-Generation for Programming Exercises With GPT-4","publication_year":2024,"publication_date":"2024-07-03","ids":{"openalex":"https://openalex.org/W4392617381","doi":"https://doi.org/10.1145/3649217.3653594"},"language":"en","primary_location":{"id":"doi:10.1145/3649217.3653594","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649217.3653594","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2403.04449","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081217838","display_name":"Imen Azaiz","orcid":"https://orcid.org/0009-0005-6458-4169"},"institutions":[{"id":"https://openalex.org/I3018771216","display_name":"LMU Klinikum","ror":"https://ror.org/02jet3w32","country_code":"DE","type":"healthcare","lineage":["https://openalex.org/I3018771216","https://openalex.org/I8204097"]},{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Imen Azaiz","raw_affiliation_strings":["LMU Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"LMU Munich, Munich, Germany","institution_ids":["https://openalex.org/I3018771216","https://openalex.org/I8204097"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039250881","display_name":"Natalie Kiesler","orcid":"https://orcid.org/0000-0002-6843-2729"},"institutions":[{"id":"https://openalex.org/I4210100612","display_name":"Georg Simon Ohm University of Applied Sciences Nuremberg","ror":"https://ror.org/00nggaz43","country_code":"DE","type":"education","lineage":["https://openalex.org/I4210100612"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Natalie Kiesler","raw_affiliation_strings":["Nuremberg Tech, Nuremberg, Germany"],"affiliations":[{"raw_affiliation_string":"Nuremberg Tech, Nuremberg, Germany","institution_ids":["https://openalex.org/I4210100612"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074181900","display_name":"Sven Strickroth","orcid":"https://orcid.org/0000-0002-9647-300X"},"institutions":[{"id":"https://openalex.org/I3018771216","display_name":"LMU Klinikum","ror":"https://ror.org/02jet3w32","country_code":"DE","type":"healthcare","lineage":["https://openalex.org/I3018771216","https://openalex.org/I8204097"]},{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sven Strickroth","raw_affiliation_strings":["LMU Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"LMU Munich, Munich, Germany","institution_ids":["https://openalex.org/I3018771216","https://openalex.org/I8204097"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5081217838"],"corresponding_institution_ids":["https://openalex.org/I3018771216","https://openalex.org/I8204097"],"apc_list":null,"apc_paid":null,"fwci":17.0335,"has_fulltext":true,"cited_by_count":48,"citation_normalized_percentile":{"value":0.99352691,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"31","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.916700005531311,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.916700005531311,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.48990100622177124}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48990100622177124}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3649217.3653594","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649217.3653594","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2403.04449","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.04449","pdf_url":"https://arxiv.org/pdf/2403.04449","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:kobv.de-opus4-th-nuernberg:2818","is_oa":false,"landing_page_url":"https://opus4.kobv.de/opus4-ohm/frontdoor/index/index/docId/2818","pdf_url":null,"source":{"id":"https://openalex.org/S4306401730","display_name":"OHMdok (Technische Hochschule N\u00fcrnberg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210100612","host_organization_name":"Georg Simon Ohm University of Applied Sciences Nuremberg","host_organization_lineage":["https://openalex.org/I4210100612"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"doc-type:conferenceObject"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2403.04449","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.04449","pdf_url":"https://arxiv.org/pdf/2403.04449","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8399999737739563,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G352791218","display_name":null,"funder_award_id":"(BMBF)","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"}],"funders":[{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4392617381.pdf"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W949889890","https://openalex.org/W1979290264","https://openalex.org/W2058580210","https://openalex.org/W2135995390","https://openalex.org/W2168795140","https://openalex.org/W2469877382","https://openalex.org/W2534043726","https://openalex.org/W2551949838","https://openalex.org/W2894540915","https://openalex.org/W2910880405","https://openalex.org/W3000240292","https://openalex.org/W3033406705","https://openalex.org/W3033498077","https://openalex.org/W3128345447","https://openalex.org/W3134953867","https://openalex.org/W4211263275","https://openalex.org/W4283705032","https://openalex.org/W4285114984","https://openalex.org/W4308205574","https://openalex.org/W4313288661","https://openalex.org/W4317502110","https://openalex.org/W4323033692","https://openalex.org/W4323033814","https://openalex.org/W4366808937","https://openalex.org/W4379539745","https://openalex.org/W4382652868","https://openalex.org/W4382654294","https://openalex.org/W4386044264","https://openalex.org/W4389762775","https://openalex.org/W4390315357","https://openalex.org/W4390490752","https://openalex.org/W4390621504"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W4391913857","https://openalex.org/W2350741829","https://openalex.org/W2530322880"],"abstract_inverted_index":{"Ever":[0],"since":[1],"Large":[2],"Language":[3],"Models":[4],"(LLMs)":[5],"and":[6,21,32,52,57,78,93,118,129,145,205,215],"related":[7],"applications":[8,221],"have":[9,35],"become":[10],"broadly":[11],"available,":[12],"several":[13],"studies":[14],"investigated":[15],"their":[16],"potential":[17],"for":[18,70,100],"assisting":[19],"educators":[20],"supporting":[22],"students":[23,47,217],"in":[24,39,122,155],"higher":[25],"education.":[26],"LLMs":[27],"such":[28,180],"as":[29,82,181],"Codex,":[30],"GPT-3.5,":[31,132],"GPT":[33],"4":[34],"shown":[36],"promising":[37],"results":[38],"the":[40,63,74,123,140,162,166,169,173,184],"context":[41],"of":[42,65,131,168,201],"large":[43],"programming":[44,75,89,106],"courses,":[45],"where":[46],"can":[48,149],"benefit":[49],"from":[50,86],"feedback":[51,99,163,177],"hints":[53],"if":[54],"provided":[55],"timely":[56],"at":[58],"scale.":[59],"This":[60],"paper":[61],"explores":[62],"quality":[64],"GPT-4":[66,94,133,147],"Turbo's":[67],"generated":[68],"output":[69,109,141,167],"prompts":[71],"containing":[72],"both":[73],"task":[76],"specification":[77],"a":[79],"student's":[80],"submission":[81,185],"input.":[83],"Two":[84],"assignments":[85],"an":[87,189],"introductory":[88],"course":[90],"were":[91],"selected,":[92],"was":[95,110,178],"asked":[96],"to":[97,126,192,207],"generate":[98],"55":[101],"randomly":[102],"chosen,":[103],"authentic":[104],"student":[105,156,170],"submissions.":[107],"The":[108,195],"qualitatively":[111],"analyzed":[112],"regarding":[113],"correctness,":[114],"personalization,":[115],"fault":[116],"localization,":[117],"other":[119],"features":[120],"identified":[121],"material.":[124],"Compared":[125],"prior":[127],"work":[128,197],"analyses":[130],"Turbo":[134,148],"shows":[135],"notable":[136],"improvements.":[137],"For":[138],"example,":[139],"is":[142,186],"more":[143],"structured":[144],"consistent.":[146],"also":[150,164],"accurately":[151],"identify":[152],"invalid":[153],"casing":[154],"programs'":[157],"output.":[158],"In":[159],"some":[160],"cases,":[161],"includes":[165],"program.":[171],"At":[172],"same":[174],"time,":[175],"inconsistent":[176],"noted":[179],"stating":[182],"that":[183],"correct":[187],"but":[188],"error":[190],"needs":[191],"be":[193],"fixed.":[194],"present":[196],"increases":[198],"our":[199],"understanding":[200],"LLMs'":[202],"potential,":[203],"limitations,":[204],"how":[206],"integrate":[208],"them":[209],"into":[210],"e-assessment":[211],"systems,":[212],"pedagogical":[213],"scenarios,":[214],"instructing":[216],"who":[218],"are":[219],"using":[220],"based":[222],"on":[223],"GPT-4.":[224]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":38},{"year":2024,"cited_by_count":7}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}