{"id":"https://openalex.org/W4411523078","doi":"https://doi.org/10.1145/3728894","title":"LLM Hallucinations in Practical Code Generation: Phenomena, Mechanism, and Mitigation","display_name":"LLM Hallucinations in Practical Code Generation: Phenomena, Mechanism, and Mitigation","publication_year":2025,"publication_date":"2025-06-22","ids":{"openalex":"https://openalex.org/W4411523078","doi":"https://doi.org/10.1145/3728894"},"language":"en","primary_location":{"id":"doi:10.1145/3728894","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728894","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3728894","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Ziyao Zhang","orcid":"https://orcid.org/0009-0008-8335-4299"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyao Zhang","raw_affiliation_strings":["Sun Yat-sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0009-0008-8335-4299","affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100329466","display_name":"Chong Wang","orcid":"https://orcid.org/0000-0003-1424-6290"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Chong Wang","raw_affiliation_strings":["Nanyang Technological University, Nanyang Avenue, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-1424-6290","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Nanyang Avenue, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100350708","display_name":"Yanlin Wang","orcid":"https://orcid.org/0000-0001-7761-7269"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanlin Wang","raw_affiliation_strings":["Sun Yat-sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0001-7761-7269","affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028655969","display_name":"Ensheng Shi","orcid":"https://orcid.org/0000-0002-5543-2025"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ensheng Shi","raw_affiliation_strings":["Huawei Cloud Computing Technologies Co., Ltd., Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5543-2025","affiliations":[{"raw_affiliation_string":"Huawei Cloud Computing Technologies Co., Ltd., Beijing, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048525009","display_name":"Yuchi Ma","orcid":"https://orcid.org/0009-0002-3304-1389"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuchi Ma","raw_affiliation_strings":["Huawei Cloud Computing Technologies Co., Ltd., Shenzhen, China"],"raw_orcid":"https://orcid.org/0009-0002-3304-1389","affiliations":[{"raw_affiliation_string":"Huawei Cloud Computing Technologies Co., Ltd., Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019101763","display_name":"Wanjun Zhong","orcid":"https://orcid.org/0009-0007-2236-228X"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wanjun Zhong","raw_affiliation_strings":["Sun Yat-sen University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0009-0007-2236-228X","affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086118824","display_name":"Jiachi Chen","orcid":"https://orcid.org/0000-0002-0192-9992"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiachi Chen","raw_affiliation_strings":["Sun Yat-sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-0192-9992","affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057897875","display_name":"Mingzhi Mao","orcid":"https://orcid.org/0000-0001-9369-7828"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingzhi Mao","raw_affiliation_strings":["Sun Yat-sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0001-9369-7828","affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000582109","display_name":"Zibin Zheng","orcid":"https://orcid.org/0000-0002-7878-4330"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zibin Zheng","raw_affiliation_strings":["Sun Yat-sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-7878-4330","affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":146.6311,"has_fulltext":false,"cited_by_count":74,"citation_normalized_percentile":{"value":0.99989477,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"2","issue":"ISSTA","first_page":"481","last_page":"503"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9902999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.670493483543396},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.6128551363945007},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6076711416244507},{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.5093420743942261},{"id":"https://openalex.org/keywords/mainstream","display_name":"Mainstream","score":0.4727773666381836},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4493083953857422},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.35393834114074707},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.19691744446754456},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.14219531416893005},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11604061722755432},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.11203044652938843}],"concepts":[{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.670493483543396},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.6128551363945007},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6076711416244507},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.5093420743942261},{"id":"https://openalex.org/C2777617010","wikidata":"https://www.wikidata.org/wiki/Q18957","display_name":"Mainstream","level":2,"score":0.4727773666381836},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4493083953857422},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.35393834114074707},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.19691744446754456},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.14219531416893005},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11604061722755432},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.11203044652938843},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3728894","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728894","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/201469","is_oa":false,"landing_page_url":"https://hdl.handle.net/10356/201469","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Journal Article"}],"best_oa_location":{"id":"doi:10.1145/3728894","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728894","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4099999964237213,"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2028889016","https://openalex.org/W2930957955","https://openalex.org/W3198685994","https://openalex.org/W3212496002","https://openalex.org/W4221157342","https://openalex.org/W4224051134","https://openalex.org/W4226287673","https://openalex.org/W4309674289","https://openalex.org/W4315706637","https://openalex.org/W4322576533","https://openalex.org/W4366770033","https://openalex.org/W4376167329","https://openalex.org/W4386185625","https://openalex.org/W4391558462","https://openalex.org/W4402670434","https://openalex.org/W4406866655"],"related_works":["https://openalex.org/W1583826057","https://openalex.org/W2377237701","https://openalex.org/W2360099860","https://openalex.org/W4323893170","https://openalex.org/W2352463596","https://openalex.org/W2380850119","https://openalex.org/W2101450440","https://openalex.org/W2048831961","https://openalex.org/W1606349578","https://openalex.org/W4399567378"],"abstract_inverted_index":{"Code":[0],"generation":[1,28,44,102,110],"aims":[2],"to":[3,71,83,116,150],"automatically":[4],"generate":[5,36],"code":[6,27,43,65,109],"from":[7,112],"input":[8],"requirements,":[9],"significantly":[10],"enhancing":[11],"development":[12,55,98],"efficiency.":[13],"Recent":[14],"large":[15],"language":[16],"models":[17],"(LLMs)":[18],"based":[19],"approaches":[20],"have":[21],"shown":[22],"promising":[23,32],"results":[24,111],"and":[25,88,96,144],"revolutionized":[26],"task.":[29],"Despite":[30],"the":[31,42,47,67,85,108,128],"performance,":[33],"LLMs":[34,115],"often":[35],"contents":[37],"with":[38],"hallucinations,":[39,131],"especially":[40],"for":[41],"scenario":[45],"requiring":[46],"handling":[48],"of":[49,90,121,130,142],"complex":[50,97],"contextual":[51],"dependencies":[52],"in":[53,63,100,163],"practical":[54,95],"process.":[56],"Although":[57],"previous":[58],"study":[59,68,82,84],"has":[60],"analyzed":[61],"hallucinations":[62,92,143],"LLM-powered":[64],"generation,":[66],"is":[69],"limited":[70],"standalone":[72],"function":[73],"generation.":[74],"In":[75],"this":[76],"paper,":[77],"we":[78,105,125,153],"conduct":[79],"an":[80,155],"empirical":[81],"phenomena,":[86],"mechanism,":[87],"mitigation":[89,157],"LLM":[91],"within":[93],"more":[94],"contexts":[99],"repository-level":[101],"scenario.":[103],"First,":[104],"manually":[106],"examine":[107],"six":[113],"mainstream":[114],"establish":[117],"a":[118],"hallucination":[119],"taxonomy":[120],"LLM-generated":[122],"code.":[123],"Next,":[124],"elaborate":[126],"on":[127],"phenomenon":[129],"analyze":[132,140],"their":[133],"distribution":[134],"across":[135],"different":[136],"models.":[137],"We":[138],"then":[139],"causes":[141],"identify":[145],"four":[146],"potential":[147],"factors":[148],"contributing":[149],"hallucinations.":[151],"Finally,":[152],"propose":[154],"RAG-based":[156],"method,":[158],"which":[159],"demonstrates":[160],"consistent":[161],"effectiveness":[162],"all":[164],"studied":[165],"LLMs.":[166]},"counts_by_year":[{"year":2026,"cited_by_count":43},{"year":2025,"cited_by_count":31}],"updated_date":"2026-07-21T08:15:58.654021","created_date":"2025-10-10T00:00:00"}