{"id":"https://openalex.org/W4396916202","doi":"https://doi.org/10.1007/s40593-024-00406-0","title":"Evaluation of LLM Tools for Feedback Generation in a Course on Concurrent Programming","display_name":"Evaluation of LLM Tools for Feedback Generation in a Course on Concurrent Programming","publication_year":2024,"publication_date":"2024-05-15","ids":{"openalex":"https://openalex.org/W4396916202","doi":"https://doi.org/10.1007/s40593-024-00406-0"},"language":"en","primary_location":{"id":"doi:10.1007/s40593-024-00406-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00406-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00406-0.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00406-0.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020340150","display_name":"Iria Est\u00e9vez Ayres","orcid":"https://orcid.org/0000-0002-1047-5398"},"institutions":[{"id":"https://openalex.org/I50357001","display_name":"Universidad Carlos III de Madrid","ror":"https://ror.org/03ths8210","country_code":"ES","type":"education","lineage":["https://openalex.org/I50357001"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Iria Est\u00e9vez-Ayres","raw_affiliation_strings":["Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain","institution_ids":["https://openalex.org/I50357001"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030782244","display_name":"Patricia Callejo","orcid":"https://orcid.org/0000-0001-6124-6213"},"institutions":[{"id":"https://openalex.org/I50357001","display_name":"Universidad Carlos III de Madrid","ror":"https://ror.org/03ths8210","country_code":"ES","type":"education","lineage":["https://openalex.org/I50357001"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Patricia Callejo","raw_affiliation_strings":["Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain","institution_ids":["https://openalex.org/I50357001"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019758363","display_name":"Miguel \u00c1ngel Hombrados-Herrera","orcid":"https://orcid.org/0000-0002-8254-8795"},"institutions":[{"id":"https://openalex.org/I50357001","display_name":"Universidad Carlos III de Madrid","ror":"https://ror.org/03ths8210","country_code":"ES","type":"education","lineage":["https://openalex.org/I50357001"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Miguel \u00c1ngel Hombrados-Herrera","raw_affiliation_strings":["Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain","institution_ids":["https://openalex.org/I50357001"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017626445","display_name":"Carlos Alario\u2010Hoyos","orcid":"https://orcid.org/0000-0002-3082-0814"},"institutions":[{"id":"https://openalex.org/I50357001","display_name":"Universidad Carlos III de Madrid","ror":"https://ror.org/03ths8210","country_code":"ES","type":"education","lineage":["https://openalex.org/I50357001"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Carlos Alario-Hoyos","raw_affiliation_strings":["Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain","institution_ids":["https://openalex.org/I50357001"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082851320","display_name":"Carlos Delgado Kloos","orcid":"https://orcid.org/0000-0003-4093-3705"},"institutions":[{"id":"https://openalex.org/I50357001","display_name":"Universidad Carlos III de Madrid","ror":"https://ror.org/03ths8210","country_code":"ES","type":"education","lineage":["https://openalex.org/I50357001"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Carlos Delgado Kloos","raw_affiliation_strings":["Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Telematic Engineering, Universidad Carlos III de Madrid, Avda. de la Universidad, 30, Legan\u00e9s, 28911, Madrid, Spain","institution_ids":["https://openalex.org/I50357001"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5020340150"],"corresponding_institution_ids":["https://openalex.org/I50357001"],"apc_list":{"value":2190,"currency":"EUR","value_usd":2790},"apc_paid":{"value":2190,"currency":"EUR","value_usd":2790},"fwci":13.9316,"has_fulltext":false,"cited_by_count":35,"citation_normalized_percentile":{"value":0.98795374,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"35","issue":"2","first_page":"774","last_page":"790"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9789000153541565,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6717709302902222},{"id":"https://openalex.org/keywords/course","display_name":"Course (navigation)","score":0.562388002872467},{"id":"https://openalex.org/keywords/educational-technology","display_name":"Educational technology","score":0.47020870447158813},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.42483091354370117},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.40772899985313416},{"id":"https://openalex.org/keywords/medical-education","display_name":"Medical education","score":0.3392786383628845},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.33319616317749023},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2551172375679016},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.15193143486976624},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1398889124393463}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6717709302902222},{"id":"https://openalex.org/C2777552389","wikidata":"https://www.wikidata.org/wiki/Q1962728","display_name":"Course (navigation)","level":2,"score":0.562388002872467},{"id":"https://openalex.org/C16443162","wikidata":"https://www.wikidata.org/wiki/Q1068473","display_name":"Educational technology","level":2,"score":0.47020870447158813},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.42483091354370117},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.40772899985313416},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.3392786383628845},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.33319616317749023},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2551172375679016},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.15193143486976624},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1398889124393463},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s40593-024-00406-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00406-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00406-0.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s40593-024-00406-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-024-00406-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-024-00406-0.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8799999952316284,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4396916202.pdf"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W55478891","https://openalex.org/W777621473","https://openalex.org/W1964718922","https://openalex.org/W1978476423","https://openalex.org/W2011607672","https://openalex.org/W2034256728","https://openalex.org/W2035650034","https://openalex.org/W2056366613","https://openalex.org/W2064754950","https://openalex.org/W2070405152","https://openalex.org/W2085906238","https://openalex.org/W2090306244","https://openalex.org/W2095907159","https://openalex.org/W2101161997","https://openalex.org/W2115268776","https://openalex.org/W2142995122","https://openalex.org/W2151828850","https://openalex.org/W2154041216","https://openalex.org/W2344269077","https://openalex.org/W2475115935","https://openalex.org/W2560140854","https://openalex.org/W2567853407","https://openalex.org/W2794062120","https://openalex.org/W2894540915","https://openalex.org/W2898125173","https://openalex.org/W2908797621","https://openalex.org/W2989864741","https://openalex.org/W3023285645","https://openalex.org/W3047890686","https://openalex.org/W3113010424","https://openalex.org/W3113108648","https://openalex.org/W3121259022","https://openalex.org/W3184396411","https://openalex.org/W3211022409","https://openalex.org/W3213654060","https://openalex.org/W4213421742","https://openalex.org/W4281989508","https://openalex.org/W4283705032","https://openalex.org/W4285782968","https://openalex.org/W4291476001","https://openalex.org/W4365515834","https://openalex.org/W4366403195","https://openalex.org/W4379878854","https://openalex.org/W4380079217","https://openalex.org/W4381586920","https://openalex.org/W4384519236","https://openalex.org/W4393147129","https://openalex.org/W4393161262","https://openalex.org/W6739901393"],"related_works":["https://openalex.org/W4318717247","https://openalex.org/W2267656092","https://openalex.org/W2317874921","https://openalex.org/W2736755255","https://openalex.org/W4362648715","https://openalex.org/W2009385780","https://openalex.org/W3082623308","https://openalex.org/W3120503679","https://openalex.org/W2359390960","https://openalex.org/W3180276870"],"abstract_inverted_index":{"Abstract":[0],"The":[1,15,65,85],"emergence":[2],"of":[3,17,51,67,90,137,151,164],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"has":[8,24],"marked":[9],"a":[10,61],"significant":[11],"change":[12],"in":[13,60,71,160,172],"education.":[14],"appearance":[16],"these":[18,138,177],"LLMs":[19,52,152],"and":[20,31,99],"their":[21,34,173],"associated":[22],"chatbots":[23,53],"yielded":[25],"several":[26],"advantages":[27],"for":[28,39,78,110],"both":[29,168],"students":[30,81],"educators,":[32],"including":[33],"use":[35],"as":[36,115],"teaching":[37],"assistants":[38],"content":[40],"creation":[41],"or":[42,118],"summarisation.":[43],"This":[44],"paper":[45],"aims":[46],"to":[47,54,80,105,122,133,175],"evaluate":[48,106,176],"the":[49,68,76,123,144,147,154],"capacity":[50],"provide":[55],"feedback":[56,79],"on":[57],"student":[58],"exercises":[59,91,145,179],"university":[62],"programming":[63,69],"course.":[64],"complexity":[66],"topic":[70],"this":[72],"study":[73],"(concurrency)":[74],"makes":[75],"need":[77],"even":[82],"more":[83],"important.":[84],"authors":[86],"conducted":[87],"an":[88,161],"assessment":[89],"submitted":[92],"by":[93,127],"students.":[94],"Then,":[95],"ChatGPT":[96],"(from":[97,101],"OpenAI)":[98],"Bard":[100],"Google)":[102],"were":[103],"employed":[104],"each":[107],"exercise,":[108],"looking":[109],"typical":[111,183],"concurrency":[112,184],"errors,":[113],"such":[114],"starvation,":[116],"deadlocks,":[117],"race":[119],"conditions.":[120],"Compared":[121],"ground-truth":[124],"evaluations":[125],"performed":[126],"expert":[128],"teachers,":[129],"it":[130],"is":[131],"possible":[132],"conclude":[134],"that":[135,167],"none":[136],"two":[139],"tools":[140,169],"can":[141],"accurately":[142],"assess":[143],"despite":[146],"generally":[148],"positive":[149],"reception":[150],"within":[153],"educational":[155],"sector.":[156],"All":[157],"attempts":[158],"result":[159],"accuracy":[162],"rate":[163],"50%,":[165],"meaning":[166],"have":[170],"limitations":[171],"ability":[174],"particular":[178],"effectively,":[180],"specifically":[181],"finding":[182],"errors.":[185]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":8}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}