{"id":"https://openalex.org/W4398239273","doi":"https://doi.org/10.1145/3639478.3643081","title":"Imperfect Code Generation: Uncovering Weaknesses in Automatic Code Generation by Large Language Models","display_name":"Imperfect Code Generation: Uncovering Weaknesses in Automatic Code Generation by Large Language Models","publication_year":2024,"publication_date":"2024-04-14","ids":{"openalex":"https://openalex.org/W4398239273","doi":"https://doi.org/10.1145/3639478.3643081"},"language":"en","primary_location":{"id":"doi:10.1145/3639478.3643081","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639478.3643081","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643081","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM 46th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643081","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111296074","display_name":"Xiaoli Lian","orcid":"https://orcid.org/0000-0002-6100-7068"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoli Lian","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6100-7068","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021796709","display_name":"Shuaisong Wang","orcid":"https://orcid.org/0009-0009-2033-7026"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuaisong Wang","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-2033-7026","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026139710","display_name":"Jieping Ma","orcid":"https://orcid.org/0009-0005-6756-8316"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jieping Ma","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0005-6756-8316","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055158271","display_name":"Xin Tan","orcid":"https://orcid.org/0000-0003-1099-3336"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Tan","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1099-3336","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060027879","display_name":"Fang Liu","orcid":"https://orcid.org/0000-0002-3905-8133"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fang Liu","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3905-8133","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014381743","display_name":"Lin Shi","orcid":"https://orcid.org/0000-0003-1476-7213"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Shi","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1476-7213","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061699251","display_name":"Cuiyun Gao","orcid":"https://orcid.org/0000-0003-4774-2434"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cuiyun Gao","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-4774-2434","affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114585361","display_name":"Li Zhang","orcid":"https://orcid.org/0000-0002-2258-5893"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Zhang","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2258-5893","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"422","last_page":"423"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9855999946594238,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9855999946594238,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9815000295639038,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9750999808311462,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.8049705624580383},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6530045866966248},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5532151460647583},{"id":"https://openalex.org/keywords/imperfect","display_name":"Imperfect","score":0.49441957473754883},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.47137579321861267},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4495203197002411},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.42752954363822937},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.14429208636283875},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.13332629203796387},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.12790939211845398},{"id":"https://openalex.org/keywords/ecology","display_name":"Ecology","score":0.06838792562484741}],"concepts":[{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.8049705624580383},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6530045866966248},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5532151460647583},{"id":"https://openalex.org/C2780310539","wikidata":"https://www.wikidata.org/wiki/Q12547192","display_name":"Imperfect","level":2,"score":0.49441957473754883},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.47137579321861267},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4495203197002411},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.42752954363822937},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.14429208636283875},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13332629203796387},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.12790939211845398},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.06838792562484741},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3639478.3643081","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639478.3643081","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643081","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM 46th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3639478.3643081","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639478.3643081","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643081","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM 46th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4398239273.pdf"},"referenced_works_count":2,"referenced_works":["https://openalex.org/W3154463802","https://openalex.org/W4385562549"],"related_works":["https://openalex.org/W2374250903","https://openalex.org/W4295769391","https://openalex.org/W2972220648","https://openalex.org/W2332667808","https://openalex.org/W1997921863","https://openalex.org/W1546413948","https://openalex.org/W3112960490","https://openalex.org/W2263832889","https://openalex.org/W2243884323","https://openalex.org/W42072456"],"abstract_inverted_index":{"The":[0],"task":[1],"of":[2,33,101,119],"code":[3,21,45,93],"generation":[4,46,94],"has":[5],"received":[6],"significant":[7],"attention":[8],"in":[9,38,43,122],"recent":[10],"years,":[11],"especially":[12],"when":[13],"the":[14,39,52,83,117,123],"pre-trained":[15],"large":[16],"language":[17],"models":[18],"(LLMs)":[19],"for":[20],"have":[22],"consistently":[23],"achieved":[24],"state-of-the-art":[25,88],"performance.":[26],"However,":[27],"there":[28],"is":[29],"currently":[30],"a":[31,34,78],"lack":[32],"comprehensive":[35],"weakness":[36],"taxonomy":[37],"field,":[40],"uncovering":[41],"weaknesses":[42,84,102],"automatic":[44],"by":[47],"LLMs.":[48],"This":[49],"may":[50],"lead":[51],"community":[53],"to":[54,113],"invest":[55],"excessive":[56],"efforts":[57],"into":[58],"well-known":[59],"hotspots":[60],"while":[61],"neglecting":[62],"many":[63],"crucial":[64],"yet":[65],"unrecognized":[66],"issues":[67],"that":[68],"deserve":[69],"more":[70],"attention.":[71],"To":[72],"bridge":[73],"this":[74],"gap,":[75],"we":[76],"conduct":[77],"systematic":[79],"study":[80,97],"on":[81,86],"analyzing":[82],"based":[85],"three":[87,91],"LLMs":[89],"across":[90,107],"widely-used":[92],"datasets.":[95],"Our":[96],"identifies":[98],"eight":[99],"types":[100],"and":[103,110,115],"assesses":[104],"their":[105],"prevalence":[106],"each":[108],"LLM":[109],"dataset,":[111],"aiming":[112],"inform":[114],"shape":[116],"trajectory":[118],"future":[120],"research":[121],"domain.":[124]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}