{"id":"https://openalex.org/W4406892560","doi":"https://doi.org/10.1109/fllm63129.2024.10852439","title":"Evaluating Large Language Models for Code Generation: Assessing Accuracy, Quality, and Performance","display_name":"Evaluating Large Language Models for Code Generation: Assessing Accuracy, Quality, and Performance","publication_year":2024,"publication_date":"2024-11-26","ids":{"openalex":"https://openalex.org/W4406892560","doi":"https://doi.org/10.1109/fllm63129.2024.10852439"},"language":"en","primary_location":{"id":"doi:10.1109/fllm63129.2024.10852439","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fllm63129.2024.10852439","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 2nd International Conference on Foundation and Large Language Models (FLLM)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014309404","display_name":"Mohammed A. Shehab","orcid":"https://orcid.org/0000-0003-3369-8540"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Mohammed A. Shehab","raw_affiliation_strings":["Concordia University,Dept. Concordia Continuing Education,Montreal,Canada"],"affiliations":[{"raw_affiliation_string":"Concordia University,Dept. Concordia Continuing Education,Montreal,Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004183493","display_name":"Mohammad Wardat","orcid":"https://orcid.org/0009-0001-0213-725X"},"institutions":[{"id":"https://openalex.org/I177721651","display_name":"Oakland University","ror":"https://ror.org/01ythxj32","country_code":"US","type":"education","lineage":["https://openalex.org/I177721651"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammad Wardat","raw_affiliation_strings":["Oakland University,Dept. of Computer Science and Engineering,Rochester Hills,USA"],"affiliations":[{"raw_affiliation_string":"Oakland University,Dept. of Computer Science and Engineering,Rochester Hills,USA","institution_ids":["https://openalex.org/I177721651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011281489","display_name":"Safwan Omari","orcid":null},"institutions":[{"id":"https://openalex.org/I134118262","display_name":"Lewis University","ror":"https://ror.org/0376vhj74","country_code":"US","type":"education","lineage":["https://openalex.org/I134118262"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Safwan Omari","raw_affiliation_strings":["Lewis University,Dept. of ECaMS,Romeoville,USA"],"affiliations":[{"raw_affiliation_string":"Lewis University,Dept. of ECaMS,Romeoville,USA","institution_ids":["https://openalex.org/I134118262"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032187551","display_name":"Yaser Jararweh","orcid":"https://orcid.org/0000-0002-4403-3846"},"institutions":[{"id":"https://openalex.org/I156983542","display_name":"Jordan University of Science and Technology","ror":"https://ror.org/03y8mtb59","country_code":"JO","type":"education","lineage":["https://openalex.org/I156983542"]}],"countries":["JO"],"is_corresponding":false,"raw_author_name":"Yaser Jararweh","raw_affiliation_strings":["Jordan University of Science and Technology,Dept. of Computer Science,Irbid,Jordan"],"affiliations":[{"raw_affiliation_string":"Jordan University of Science and Technology,Dept. of Computer Science,Irbid,Jordan","institution_ids":["https://openalex.org/I156983542"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5014309404"],"corresponding_institution_ids":["https://openalex.org/I60158472"],"apc_list":null,"apc_paid":null,"fwci":1.9896,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.89017382,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"407","last_page":"416"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9549000263214111,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9028000235557556,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8024119138717651},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.512237012386322},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4966018795967102},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.44835904240608215},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.3682120144367218},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3583013415336609},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.11068272590637207},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08445620536804199},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.06885197758674622}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8024119138717651},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.512237012386322},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4966018795967102},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.44835904240608215},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3682120144367218},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3583013415336609},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.11068272590637207},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08445620536804199},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.06885197758674622},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/fllm63129.2024.10852439","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fllm63129.2024.10852439","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 2nd International Conference on Foundation and Large Language Models (FLLM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6200000047683716,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1964962870","https://openalex.org/W2112409390","https://openalex.org/W2117603018","https://openalex.org/W2281037989","https://openalex.org/W4281763794","https://openalex.org/W4323033785","https://openalex.org/W4396723256","https://openalex.org/W4398239202","https://openalex.org/W4399577158","https://openalex.org/W4399577572","https://openalex.org/W6679096250","https://openalex.org/W6779831603","https://openalex.org/W6798182279","https://openalex.org/W6808597555","https://openalex.org/W6810081322"],"related_works":["https://openalex.org/W4376453582","https://openalex.org/W3147033875","https://openalex.org/W1568390478","https://openalex.org/W2764722704","https://openalex.org/W4231937131","https://openalex.org/W82714704","https://openalex.org/W1519020487","https://openalex.org/W2382583096","https://openalex.org/W147463599","https://openalex.org/W1835076629"],"abstract_inverted_index":{"Large":[0],"Language":[1,36],"Models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"utilized":[6],"for":[7],"software":[8],"engineering":[9],"tasks,":[10],"including":[11],"code":[12,21,33,50,82],"generation.":[13],"While":[14],"prior":[15],"research":[16],"has":[17],"primarily":[18],"focused":[19,74],"on":[20,54,75],"completion,":[22],"there":[23],"remains":[24],"a":[25,115],"gap":[26],"in":[27,48],"understanding":[28],"LLMs\u2019":[29,46,70],"ability":[30,91],"to":[31,67,92],"generate":[32,93],"from":[34,51],"Natural":[35],"Processing":[37],"(NLP)":[38],"descriptions.":[39,57],"In":[40],"this":[41],"study,":[42],"we":[43],"investigated":[44],"three":[45,60],"performance":[47,87,113,137],"generating":[49],"scratch":[52],"based":[53],"NLP":[55],"task":[56],"We":[58],"employed":[59],"evaluation":[61],"levels,":[62],"Accuracy,":[63],"Quality,":[64],"and":[65,78,84,86,98,106],"Performance,":[66],"assess":[68],"the":[69,89,111],"results.":[71],"Accuracy":[72],"metrics":[73],"error":[76,101],"types":[77],"counts,":[79],"quality":[80],"assessed":[81],"readability":[83],"maintainability,":[85],"measured":[88],"models\u2019":[90],"optimized":[94],"solutions.":[95],"Codex,":[96],"Copilot,":[97],"PaLM2":[99],"exhibited":[100],"rates":[102],"of":[103,125,131],"9.68%,":[104],"32.25%,":[105],"48.38%,":[107],"respectively.":[108],"Copilot":[109],"showed":[110],"best":[112],"with":[114,123,128],"higher":[116],"maintainability":[117],"index.":[118],"However,":[119],"all":[120],"LLMs":[121],"struggled":[122],"tasks":[124],"high":[126],"complexity,":[127],"time":[129],"complexities":[130],"O(m\u2022n\u2022log(n)),":[132],"as":[133],"evidenced":[134],"by":[135],"their":[136],"metrics.":[138]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3}],"updated_date":"2026-04-23T09:07:50.710637","created_date":"2025-10-10T00:00:00"}