{"id":"https://openalex.org/W7160911977","doi":"https://doi.org/10.1016/j.infsof.2026.108185","title":"An overview of evaluation and enhancement methods for code generation by large language models","display_name":"An overview of evaluation and enhancement methods for code generation by large language models","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7160911977","doi":"https://doi.org/10.1016/j.infsof.2026.108185"},"language":"en","primary_location":{"id":"doi:10.1016/j.infsof.2026.108185","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108185","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.infsof.2026.108185","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135962532","display_name":"Jacob Truong","orcid":"https://orcid.org/0009-0003-2653-3286"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Jacob Truong","raw_affiliation_strings":["Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia"],"raw_orcid":"https://orcid.org/0009-0003-2653-3286","affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135919296","display_name":"Van Nguyen","orcid":null},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Van Nguyen","raw_affiliation_strings":["Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5838-3409","affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085593383","display_name":"Thanh Thi Nguyen","orcid":"https://orcid.org/0000-0001-9709-1663"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Van Nguyen","raw_affiliation_strings":["Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5838-3409","affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"last","author":{"id":null,"display_name":"Thanh Thi Nguyen","orcid":"https://orcid.org/0000-0001-9709-1663"},"institutions":[{"id":"https://openalex.org/I174025329","display_name":"University of the Sunshine Coast","ror":"https://ror.org/016gb9e15","country_code":"AU","type":"education","lineage":["https://openalex.org/I174025329"]},{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Thanh Thi Nguyen","raw_affiliation_strings":["Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia","School of Science, Technology and Engineering, University of the Sunshine Coast, Petrie, QLD, 4502, Australia"],"raw_orcid":"https://orcid.org/0000-0001-9709-1663","affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Monash University, Clayton, VIC, 3800, Australia","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"School of Science, Technology and Engineering, University of the Sunshine Coast, Petrie, QLD, 4502, Australia","institution_ids":["https://openalex.org/I174025329"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5135919296"],"corresponding_institution_ids":["https://openalex.org/I174025329","https://openalex.org/I56590836"],"apc_list":{"value":3350,"currency":"USD","value_usd":3350},"apc_paid":{"value":3350,"currency":"USD","value_usd":3350},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.55220018,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"197","issue":null,"first_page":"108185","last_page":"108185"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5619000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5619000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.12250000238418579,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.05530000105500221,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.5989999771118164},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.5631999969482422},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4708999991416931},{"id":"https://openalex.org/keywords/software-quality-assurance","display_name":"Software quality assurance","score":0.46639999747276306},{"id":"https://openalex.org/keywords/maintainability","display_name":"Maintainability","score":0.44859999418258667},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4235000014305115},{"id":"https://openalex.org/keywords/dimension","display_name":"Dimension (graph theory)","score":0.4156999886035919},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.39250001311302185},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.38339999318122864}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7067000269889832},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.5989999771118164},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.5631999969482422},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.542900025844574},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4708999991416931},{"id":"https://openalex.org/C2776969324","wikidata":"https://www.wikidata.org/wiki/Q613918","display_name":"Software quality assurance","level":5,"score":0.46639999747276306},{"id":"https://openalex.org/C160713754","wikidata":"https://www.wikidata.org/wiki/Q1389965","display_name":"Maintainability","level":2,"score":0.44859999418258667},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4235000014305115},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.4156999886035919},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.39250001311302185},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.38339999318122864},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.3799000084400177},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37389999628067017},{"id":"https://openalex.org/C144559511","wikidata":"https://www.wikidata.org/wiki/Q2986279","display_name":"Principal (computer security)","level":2,"score":0.37070000171661377},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.33869999647140503},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.32989999651908875},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.3199000060558319},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.29789999127388},{"id":"https://openalex.org/C62913178","wikidata":"https://www.wikidata.org/wiki/Q7554361","display_name":"Software security assurance","level":4,"score":0.2955999970436096},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.2953000068664551},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2912999987602234},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2863999903202057},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.2858999967575073},{"id":"https://openalex.org/C75606506","wikidata":"https://www.wikidata.org/wiki/Q1049183","display_name":"Formal methods","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.27549999952316284},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.27230000495910645},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.26750001311302185},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.infsof.2026.108185","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108185","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.infsof.2026.108185","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108185","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.5394233465194702,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1964962870","https://openalex.org/W2081917329","https://openalex.org/W2108025693","https://openalex.org/W2158864412","https://openalex.org/W3005855585","https://openalex.org/W3163122677","https://openalex.org/W4242878916","https://openalex.org/W4311887664","https://openalex.org/W4322766882","https://openalex.org/W4366204357","https://openalex.org/W4401508141","https://openalex.org/W4405908042","https://openalex.org/W4406602186","https://openalex.org/W4406688098","https://openalex.org/W4415589175"],"related_works":[],"abstract_inverted_index":{"Context:":[0],"Recent":[1],"advances":[2],"in":[3,175,191,283],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"have":[8,198],"led":[9],"to":[10,32,43,59,79,83,114],"the":[11,36,47,50,61,81,97,117,126,145,253,259,273],"rapid":[12],"deployment":[13],"of":[14,19,35,75,125,220,255,261,275],"automated":[15],"generation":[16],"tools":[17,31],"capable":[18],"producing":[20],"source":[21],"code.":[22],"As":[23],"these":[24,182,225],"models":[25,48],"increasingly":[26],"transition":[27],"from":[28,144],"being":[29],"experimental":[30],"established":[33],"elements":[34],"software":[37,127,227],"development,":[38],"a":[39,122,138,169,188,218,234],"critical":[40,226],"question":[41,85],"arises:":[42],"what":[44],"extent":[45],"do":[46],"and":[49,91,96,110,161,178,195,202,208,217,242,245,267,271],"code":[51,98,105,139,269],"they":[52,99],"generate":[53],"satisfy,":[54,60],"or":[55,130],"can":[56],"be":[57],"made":[58],"rigorous,":[62],"multifaceted":[63],"quality":[64,106,128,140,228],"standards":[65],"required":[66],"for":[67,94,224,237,278],"professional,":[68],"real-world":[69],"engineering?":[70],"Objective:":[71],"The":[72,231],"primary":[73],"aim":[74],"this":[76,84,165],"study":[77],"is":[78,107],"find":[80],"answer":[82],"by":[86],"exploring":[87],"existing":[88,173],"evaluation":[89,176,200,244],"frameworks":[90,177,201,277],"enhancement":[92,179],"strategies":[93,180],"LLMs":[95],"generate.":[100],"By":[101],"examining":[102],"how":[103],"generated":[104],"currently":[108],"assessed":[109],"improved,":[111],"we":[112,167],"hope":[113],"determine":[115],"if":[116,131],"current":[118],"research":[119,174,192,239],"methodologies":[120],"provide":[121],"balanced":[123],"coverage":[124],"spectrum":[129],"significant":[132],"disparities":[133],"exist.":[134],"Method:":[135],"We":[136,247],"propose":[137,248],"dimension":[141],"taxonomy":[142],"adapted":[143],"ISO/IEC":[146],"25010":[147],"standard,":[148],"encompassing":[149],"four":[150],"principal":[151],"attributes:":[152],"Functional":[153],"Correctness":[154],"(FC),":[155],"Security":[156],"(SE),":[157],"Performance":[158],"Efficiency":[159],"(PE),":[160],"Maintainability":[162],"(MA).":[163],"Using":[164],"framework,":[166],"conduct":[168],"literature":[170],"review":[171],"analysing":[172],"across":[181],"dimensions.":[183,229],"Results:":[184],"Our":[185],"analysis":[186],"reveals":[187],"substantial":[189],"imbalance":[190],"focus.":[193],"FC,":[194],"increasingly,":[196],"SE":[197],"well-established":[199],"improvement":[203,281],"strategies.":[204],"In":[205],"contrast,":[206],"PE":[207,241],"MA":[209],"remain":[210],"significantly":[211],"underexamined,":[212],"with":[213],"few":[214],"standardised":[215],"benchmarks":[216],"lack":[219],"targeted":[221],"fine-tuning":[222],"approaches":[223],"Conclusion:":[230],"survey":[232],"identifies":[233],"pressing":[235],"need":[236],"broader":[238],"into":[240],"MA-oriented":[243],"enhancement.":[246],"several":[249],"promising":[250],"directions:":[251],"(i)":[252],"creation":[254],"formal":[256],"benchmarks;":[257],"(ii)":[258],"development":[260],"reinforcement":[262],"learning":[263],"techniques":[264],"leveraging":[265],"static":[266],"dynamic":[268],"feedback;":[270],"(iii)":[272],"use":[274],"multi-agent":[276],"iterative,":[279],"critique-based":[280],"grounded":[282],"verifiable":[284],"diagnostic":[285],"artefacts.":[286]},"counts_by_year":[],"updated_date":"2026-08-06T08:24:18.245995","created_date":"2026-05-13T00:00:00"}