{"id":"https://openalex.org/W4390621504","doi":"https://doi.org/10.1109/fie58773.2023.10343457","title":"Exploring the Potential of Large Language Models to Generate Formative Programming Feedback","display_name":"Exploring the Potential of Large Language Models to Generate Formative Programming Feedback","alternate_titles":[],"publication_year":2023,"publication_date":"2023-10-18","ids":{"openalex":"https://openalex.org/W4390621504","doi":"https://doi.org/10.1109/fie58773.2023.10343457"},"language":"en","primary_location":{"id":"doi:10.1109/fie58773.2023.10343457","doi":"https://doi.org/10.1109/fie58773.2023.10343457","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fie58773.2023.10343457","pdf_url":null,"source":{"id":"https://openalex.org/S4363607793","display_name":"IEEE Frontiers in Education Conference (FIE)","issn_l":"0190-5848","issn":["0190-5848","2377-634X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Frontiers in Education Conference (FIE)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4390621504"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2309.00029","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039250881","display_name":"Natalie Kiesler","orcid":"https://orcid.org/0000-0002-6843-2729","observed_orcids":["https://orcid.org/0000-0002-6843-2729"]},"institutions":[{"id":"https://openalex.org/I4210138519","display_name":"DIPF | Leibniz Institute for Research and Information in Education","ror":"https://ror.org/0327sr118","country_code":"DE","type":"facility","lineage":["https://openalex.org/I315704651","https://openalex.org/I4210138519"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Natalie Kiesler","raw_affiliation_strings":["DIPF Leibniz Institute for Research and Information in Education,Information Center Education,Frankfurt am Main,Germany","Information Center Education, DIPF Leibniz Institute for Research and Information in Education, Frankfurt am Main, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DIPF Leibniz Institute for Research and Information in Education,Information Center Education,Frankfurt am Main,Germany","institution_ids":["https://openalex.org/I4210138519"]},{"raw_affiliation_string":"Information Center Education, DIPF Leibniz Institute for Research and Information in Education, Frankfurt am Main, Germany","institution_ids":["https://openalex.org/I4210138519"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084167961","display_name":"Dominic Lohr","orcid":"https://orcid.org/0000-0002-6330-2327","observed_orcids":["https://orcid.org/0000-0002-6330-2327"]},"institutions":[{"id":"https://openalex.org/I181369854","display_name":"Friedrich-Alexander-Universit\u00e4t Erlangen-N\u00fcrnberg","ror":"https://ror.org/00f7hpc57","country_code":"DE","type":"education","lineage":["https://openalex.org/I181369854"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Dominic Lohr","raw_affiliation_strings":["Friedrich-Alexander-University,Computer Science Education,Erlangen,Germany","Computer Science Education, Friedrich-Alexander-University, Erlangen, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Friedrich-Alexander-University,Computer Science Education,Erlangen,Germany","institution_ids":["https://openalex.org/I181369854"]},{"raw_affiliation_string":"Computer Science Education, Friedrich-Alexander-University, Erlangen, Germany","institution_ids":["https://openalex.org/I181369854"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052659996","display_name":"Hieke Keuning","orcid":"https://orcid.org/0000-0001-5778-7519","observed_orcids":["https://orcid.org/0000-0001-5778-7519"]},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Hieke Keuning","raw_affiliation_strings":["Utrecht University,Information and Computing Sciences,Utrecht,The Netherlands","Information and Computing Sciences, Utrecht University, Utrecht, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Utrecht University,Information and Computing Sciences,Utrecht,The Netherlands","institution_ids":["https://openalex.org/I193662353"]},{"raw_affiliation_string":"Information and Computing Sciences, Utrecht University, Utrecht, The Netherlands","institution_ids":["https://openalex.org/I193662353"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":25.0169,"has_fulltext":true,"cited_by_count":142,"citation_normalized_percentile":{"value":0.99538958,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.004000000189989805,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9980000257492065},{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.9190000295639038},{"id":"https://openalex.org/keywords/formative-feedback","display_name":"formative feedback","score":0.8569999933242798},{"id":"https://openalex.org/keywords/programming-education","display_name":"programming education","score":0.9039999842643738},{"id":"https://openalex.org/keywords/introductory-programming","display_name":"introductory programming","score":0.9269999861717224},{"id":"https://openalex.org/keywords/cs1","display_name":"CS1","score":0.7900000214576721},{"id":"https://openalex.org/keywords/student-errors","display_name":"student errors","score":0.6940000057220459},{"id":"https://openalex.org/keywords/feedback-types","display_name":"feedback types","score":0.7549999952316284},{"id":"https://openalex.org/keywords/novice-programmers","display_name":"novice programmers","score":0.503000020980835},{"id":"https://openalex.org/keywords/automatic-feedback","display_name":"automated feedback","score":0.3109999895095825}],"concepts":[{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.8826793432235718},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.795097827911377},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4724576771259308},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4454786479473114},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.40473100543022156},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.36230355501174927},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3304271101951599},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11178001761436462},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/fie58773.2023.10343457","doi":"https://doi.org/10.1109/fie58773.2023.10343457","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fie58773.2023.10343457","pdf_url":null,"source":{"id":"https://openalex.org/S4363607793","display_name":"IEEE Frontiers in Education Conference (FIE)","issn_l":"0190-5848","issn":["0190-5848","2377-634X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Frontiers in Education Conference (FIE)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2309.00029","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.00029","pdf_url":"https://arxiv.org/pdf/2309.00029","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dspace.library.uu.nl:1874/431620","doi":null,"is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/431620","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"/dk/atira/pure/researchoutput/researchoutputtypes/workingpaper/preprint"},{"id":"pmh:oai:kobv.de-opus4-th-nuernberg:2811","doi":null,"is_oa":false,"landing_page_url":"https://opus4.kobv.de/opus4-ohm/frontdoor/index/index/docId/2811","pdf_url":null,"source":{"id":"https://openalex.org/S4306401730","display_name":"OHMdok (Technische Hochschule N\u00fcrnberg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210100612","host_organization_name":"Georg Simon Ohm University of Applied Sciences Nuremberg","host_organization_lineage":["https://openalex.org/I4210100612"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conferenceobject"},{"id":"doi:10.48550/arxiv.2309.00029","doi":"https://doi.org/10.48550/arxiv.2309.00029","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2309.00029","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2309.00029","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.00029","pdf_url":"https://arxiv.org/pdf/2309.00029","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.9901999831199646}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4390621504.pdf","grobid_xml":"https://content.openalex.org/works/W4390621504.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2116817443","https://openalex.org/W2551949838","https://openalex.org/W2564331714","https://openalex.org/W2783082347","https://openalex.org/W2894540915","https://openalex.org/W3033498077","https://openalex.org/W3108843465","https://openalex.org/W4211263275","https://openalex.org/W4283705032","https://openalex.org/W4291476001","https://openalex.org/W4313288661","https://openalex.org/W4317502110","https://openalex.org/W4321162272","https://openalex.org/W4323033692","https://openalex.org/W4323033785","https://openalex.org/W4323037544","https://openalex.org/W4366725264","https://openalex.org/W4380353816","https://openalex.org/W4382515993","https://openalex.org/W4382654255","https://openalex.org/W4382654480","https://openalex.org/W4383175697","https://openalex.org/W4386044264","https://openalex.org/W4386099272","https://openalex.org/W4396214523","https://openalex.org/W6760294328"],"related_works":["https://openalex.org/W4390490752","https://openalex.org/W7167436716","https://openalex.org/W4407681796","https://openalex.org/W4413081703","https://openalex.org/W4412636642","https://openalex.org/W4409211903","https://openalex.org/W7125473685","https://openalex.org/W4407004714","https://openalex.org/W4410986803","https://openalex.org/W4414901525"],"abstract_inverted_index":{"Ever":[0],"since":[1],"the":[2,34,47,130,153],"emergence":[3],"of":[4,36,129],"large":[5],"language":[6],"models":[7],"(LLMs)":[8],"and":[9,17,42,80,116,134],"related":[10],"applications,":[11],"such":[12,37],"as":[13,44,105,156],"ChatGPT,":[14],"its":[15,86],"performance":[16],"error":[18],"analysis":[19],"for":[20,39,107,127,162],"programming":[21,78,95,132],"tasks":[22,133],"have":[23],"been":[24],"subject":[25],"to":[26,51,71,113,151],"research.":[27],"In":[28,58],"this":[29],"work-in-progress":[30],"paper,":[31],"we":[32,45,60,92],"explore":[33],"potential":[35],"LLMs":[38],"computing":[40],"educators":[41,145],"learners,":[43],"analyze":[46],"feedback":[48,83,115],"it":[49,157],"generates":[50],"a":[52,98,102],"given":[53],"input":[54,106],"containing":[55],"program":[56],"code.":[57],"particular,":[59],"aim":[61],"at":[62],"(1)":[63],"exploring":[64],"how":[65,150],"an":[66],"LLM":[67],"like":[68],"ChatGPT":[69,108,123],"responds":[70],"students":[72,140],"seeking":[73],"help":[74],"with":[75,110],"their":[76],"introductory":[77,131],"tasks,":[79],"(2)":[81],"identifying":[82],"types":[84],"in":[85],"responses.":[87],"To":[88],"achieve":[89],"these":[90],"goals,":[91],"used":[93],"students'":[94],"sequences":[96],"from":[97],"dataset":[99],"gathered":[100],"within":[101],"CS1":[103],"course":[104],"along":[109],"questions":[111],"required":[112],"elicit":[114],"correct":[117],"solutions.":[118],"The":[119],"results":[120],"show":[121],"that":[122,139],"performs":[124],"reasonably":[125],"well":[126],"some":[128],"student":[135],"errors,":[136],"which":[137],"means":[138],"can":[141,158],"potentially":[142],"benefit.":[143],"However,":[144],"should":[146],"provide":[147],"guidance":[148],"on":[149],"use":[152],"provided":[154],"feedback,":[155],"contain":[159],"misleading":[160],"information":[161],"novices.":[163]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4390621504","counts_by_year":[{"year":2026,"cited_by_count":42},{"year":2025,"cited_by_count":48},{"year":2024,"cited_by_count":43},{"year":2023,"cited_by_count":9}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2024-01-07T00:00:00"}
