{"id":"https://openalex.org/W4400113646","doi":"https://doi.org/10.1109/mipro60963.2024.10569820","title":"LLM Generative AI and Students\u2019 Exam Code Evaluation: Qualitative and Quantitative Analysis","display_name":"LLM Generative AI and Students\u2019 Exam Code Evaluation: Qualitative and Quantitative Analysis","publication_year":2024,"publication_date":"2024-05-20","ids":{"openalex":"https://openalex.org/W4400113646","doi":"https://doi.org/10.1109/mipro60963.2024.10569820"},"language":"en","primary_location":{"id":"doi:10.1109/mipro60963.2024.10569820","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mipro60963.2024.10569820","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 47th MIPRO ICT and Electronics Convention (MIPRO)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5094032247","display_name":"Ema Smoli\u0107","orcid":null},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"E. Smoli\u0107","raw_affiliation_strings":["University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing","institution_ids":["https://openalex.org/I181343428"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099635481","display_name":"M. Paveli\u0107","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"M. Paveli\u0107","raw_affiliation_strings":["Šibenik University of Applied Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Šibenik University of Applied Sciences","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064998229","display_name":"Bartol Boras","orcid":"https://orcid.org/0009-0007-8022-7828"},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"B. Boras","raw_affiliation_strings":["University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing","institution_ids":["https://openalex.org/I181343428"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067736091","display_name":"Igor Mekterovi\u0107","orcid":"https://orcid.org/0000-0002-9813-1935"},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"I. Mekterovi\u0107","raw_affiliation_strings":["University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing","institution_ids":["https://openalex.org/I181343428"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066895731","display_name":"Tomislav Jagu\u0161t","orcid":"https://orcid.org/0000-0001-7878-0455"},"institutions":[{"id":"https://openalex.org/I181343428","display_name":"University of Zagreb","ror":"https://ror.org/00mv6sv71","country_code":"HR","type":"education","lineage":["https://openalex.org/I181343428"]}],"countries":["HR"],"is_corresponding":false,"raw_author_name":"T. Jagu\u0161t","raw_affiliation_strings":["University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zagreb,Faculty of Electrical Engineering and Computing,Department of Applied Computing","institution_ids":["https://openalex.org/I181343428"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.0181,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.92391978,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1261","last_page":"1266"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9363999962806702,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9363999962806702,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7728219032287598},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.7207018136978149},{"id":"https://openalex.org/keywords/qualitative-analysis","display_name":"Qualitative analysis","score":0.6010096073150635},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5816472768783569},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4517842233181},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35477355122566223},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33259403705596924},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.32200735807418823},{"id":"https://openalex.org/keywords/qualitative-research","display_name":"Qualitative research","score":0.2610664665699005},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.05156224966049194}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7728219032287598},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.7207018136978149},{"id":"https://openalex.org/C3018587665","wikidata":"https://www.wikidata.org/wiki/Q7268696","display_name":"Qualitative analysis","level":3,"score":0.6010096073150635},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5816472768783569},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4517842233181},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35477355122566223},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33259403705596924},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.32200735807418823},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.2610664665699005},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.05156224966049194},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mipro60963.2024.10569820","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mipro60963.2024.10569820","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 47th MIPRO ICT and Electronics Convention (MIPRO)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.4000000059604645}],"awards":[],"funders":[{"id":"https://openalex.org/F4320335551","display_name":"Erasmus+","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W4292779060","https://openalex.org/W4322631505","https://openalex.org/W4366808937","https://openalex.org/W4379540175","https://openalex.org/W4380353816","https://openalex.org/W4382654255","https://openalex.org/W4382654294","https://openalex.org/W4384304728","https://openalex.org/W4385774908","https://openalex.org/W4386707723","https://openalex.org/W4389158474","https://openalex.org/W4389712680","https://openalex.org/W4389762775","https://openalex.org/W4389966389","https://openalex.org/W4389988584","https://openalex.org/W4390621504","https://openalex.org/W4390723636","https://openalex.org/W4391555526","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W2380075625","https://openalex.org/W4390718435","https://openalex.org/W4390549206","https://openalex.org/W3137171911","https://openalex.org/W4379540039","https://openalex.org/W4237784285","https://openalex.org/W2374712251","https://openalex.org/W4383031710","https://openalex.org/W3211753092","https://openalex.org/W2386000789"],"abstract_inverted_index":{"Since":[0],"the":[1,10,48,115],"introduction":[2],"of":[3,12,37,50,86,117],"generative":[4],"artificial":[5],"intelligence":[6],"(GAI)":[7],"technology":[8,39],"in":[9,63,108,110],"context":[11],"large":[13],"language":[14],"models":[15],"(LLMs),":[16],"it":[17],"has":[18],"been":[19],"widely":[20],"used":[21],"for":[22,41],"information":[23],"extraction":[24],"and/or":[25],"extrapolation":[26],"from":[27,74],"different":[28],"sources.":[29],"In":[30],"computer":[31],"science":[32],"education,":[33],"a":[34,75],"potential":[35],"application":[36],"such":[38,58],"is":[40],"automatic":[42],"code":[43,64,88,120],"review,":[44],"i.":[45,95],"e.":[46,96],"shifting":[47],"burden":[49],"debugging":[51],"non-compilable":[52],"code,":[53],"detecting":[54],"overlooked":[55],"optimization":[56],"concerns":[57],"as":[59,126,128],"poor":[60],"memory":[61],"management":[62],"that":[65,107],"otherwise":[66],"passes":[67],"automated":[68],"tests,":[69],"and":[70,101,124,134],"other":[71],"advanced":[72],"tasks":[73],"human":[76],"grader":[77],"to":[78],"LLMs.":[79],"However,":[80],"LLMs":[81],"are":[82],"currently":[83],"not":[84],"capable":[85],"evaluating":[87],"or":[89],"mathematical":[90],"expressions":[91],"with":[92],"100%":[93],"reliability,":[94],"beyond":[97],"token":[98],"pattern":[99],"recognition":[100],"subsequent":[102],"probabilistic":[103],"answer":[104],"generation.":[105],"With":[106],"mind,":[109],"this":[111],"paper,":[112],"we":[113],"explore":[114],"risk":[116],"incorrect":[118],"LLM":[119],"evaluation,":[121],"both":[122],"descriptive":[123],"numerical,":[125],"well":[127],"begin":[129],"research":[130],"on":[131],"its":[132],"mitigation":[133],"propose":[135],"further":[136],"work":[137],"directions.":[138]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}