{"id":"https://openalex.org/W4402500779","doi":"https://doi.org/10.48550/arxiv.2408.08333","title":"CodeMirage: Hallucinations in Code Generated by Large Language Models","display_name":"CodeMirage: Hallucinations in Code Generated by Large Language Models","publication_year":2024,"publication_date":"2024-08-14","ids":{"openalex":"https://openalex.org/W4402500779","doi":"https://doi.org/10.48550/arxiv.2408.08333"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2408.08333","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.08333","pdf_url":"https://arxiv.org/pdf/2408.08333","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2408.08333","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102953308","display_name":"Vibhor Agarwal","orcid":"https://orcid.org/0009-0000-8972-5259","observed_orcids":["https://orcid.org/0009-0000-8972-5259"]},"institutions":[{"id":"https://openalex.org/I28290843","display_name":"University of Surrey","ror":"https://ror.org/00ks66431","country_code":"GB","type":"education","lineage":["https://openalex.org/I28290843"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Agarwal, Vibhor","raw_affiliation_strings":["University of Surrey , Surrey , UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Surrey , Surrey , UK","institution_ids":["https://openalex.org/I28290843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029055424","display_name":"Yulong Pei","orcid":"https://orcid.org/0000-0003-3739-5627","observed_orcids":["https://orcid.org/0000-0003-3739-5627"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pei, Yulong","raw_affiliation_strings":["JP Morgan AI Research , London , UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JP Morgan AI Research , London , UK","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082203057","display_name":"Salwa Alamir","orcid":"https://orcid.org/0009-0006-6650-7041","observed_orcids":["https://orcid.org/0009-0006-6650-7041"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alamir, Salwa","raw_affiliation_strings":["JP Morgan AI Research , London , UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JP Morgan AI Research , London , UK","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048424819","display_name":"Xiaomo Liu","orcid":"https://orcid.org/0000-0003-4184-4202","observed_orcids":["https://orcid.org/0000-0003-4184-4202"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xiaomo","raw_affiliation_strings":["JP Morgan AI Research , New York , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JP Morgan AI Research , New York , USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":60,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.0038999998942017555,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7990000247955322},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.9300000071525574},{"id":"https://openalex.org/keywords/hallucination-detection","display_name":"hallucination detection","score":0.871999979019165},{"id":"https://openalex.org/keywords/hallucination-taxonomy","display_name":"hallucination taxonomy","score":0.8119999766349792},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.8809999823570251},{"id":"https://openalex.org/keywords/code-llama","display_name":"CodeLlama","score":0.8019999861717224},{"id":"https://openalex.org/keywords/codebert","display_name":"CodeBERT","score":0.7549999952316284},{"id":"https://openalex.org/keywords/humaneval","display_name":"HumanEval","score":0.6990000009536743},{"id":"https://openalex.org/keywords/mbpp","display_name":"MBPP","score":0.9729999899864197},{"id":"https://openalex.org/keywords/hallucination-mitigation","display_name":"hallucination mitigation","score":0.6990000009536743}],"concepts":[{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6172634959220886},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5171663165092468},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4651046693325043},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.39425015449523926},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3821302056312561},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.16596582531929016},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2408.08333","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.08333","pdf_url":"https://arxiv.org/pdf/2408.08333","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2408.08333","doi":"https://doi.org/10.48550/arxiv.2408.08333","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2408.08333","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2408.08333","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.08333","pdf_url":"https://arxiv.org/pdf/2408.08333","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402500779.pdf","grobid_xml":"https://content.openalex.org/works/W4402500779.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W3126675481","https://openalex.org/W3158419028","https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W4366733439","https://openalex.org/W4367189652","https://openalex.org/W4384918448","https://openalex.org/W4386185625","https://openalex.org/W4387928758","https://openalex.org/W4389524317","https://openalex.org/W4389599215","https://openalex.org/W4392240262","https://openalex.org/W4401066072","https://openalex.org/W4401885808","https://openalex.org/W4402860127","https://openalex.org/W4408562727","https://openalex.org/W4412158322","https://openalex.org/W7133198585","https://openalex.org/W7133231870"],"related_works":["https://openalex.org/W4409362569","https://openalex.org/W4417171404","https://openalex.org/W4403580613","https://openalex.org/W7166888659","https://openalex.org/W4411523078","https://openalex.org/W7161115734","https://openalex.org/W4400517027","https://openalex.org/W4415939046","https://openalex.org/W4408619886","https://openalex.org/W7130584174"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4,59],"shown":[5],"promising":[6],"potentials":[7],"in":[8,38,51,85,90,98,115],"program":[9],"generation":[10,87],"and":[11,88,129,166,177,191,207,230],"no-code":[12],"automation.":[13],"However,":[14],"LLMs":[15,81,182],"are":[16],"prone":[17],"to":[18,82,95,211],"generate":[19,23],"hallucinations,":[20],"i.e.,":[21],"they":[22],"text":[24,44],"which":[25],"sounds":[26],"plausible":[27],"but":[28],"is":[29,108],"incorrect.":[30],"Although":[31],"there":[32],"has":[33],"been":[34],"a":[35,130],"recent":[36],"surge":[37],"research":[39],"on":[40,204,216],"LLM":[41],"hallucinations":[42,97,114,229],"for":[43,144,156,173,227],"generation,":[45],"similar":[46],"hallucination":[47,127,135,175],"phenomenon":[48],"can":[49,58],"happen":[50],"code":[52,57,86,99,117,126,134,145,154,174,228],"generation.":[53,100],"Sometimes":[54],"the":[55,77,102,109,116,125,139,171,202,212,220],"generated":[56,118,152],"syntactical":[60],"or":[61],"logical":[62],"errors":[63],"as":[64,66,184,186,188],"well":[65,187],"more":[67],"advanced":[68],"issues":[69],"like":[70],"security":[71],"vulnerabilities,":[72],"memory":[73],"leaks,":[74],"etc.":[75],"Given":[76],"wide":[78],"adaptation":[79],"of":[80,104,133],"enhance":[83],"efficiency":[84],"development":[89],"general,":[91],"it":[92],"becomes":[93],"imperative":[94],"investigate":[96],"To":[101],"best":[103,203],"our":[105,232],"knowledge,":[106],"this":[107],"first":[110,140],"attempt":[111],"at":[112],"studying":[113],"by":[119,123],"LLMs.":[120],"We":[121,137,168,197],"start":[122],"introducing":[124],"definition":[128],"comprehensive":[131],"taxonomy":[132],"types.":[136],"propose":[138,170],"benchmark":[141,148],"CodeMirage":[142],"dataset":[143,206],"hallucinations.":[146],"The":[147],"contains":[149],"1,137":[150],"GPT-3.5":[151,190],"hallucinated":[153],"snippets":[155],"Python":[157],"programming":[158],"problems":[159],"from":[160],"two":[161],"base":[162],"datasets":[163],"-":[164],"HumanEval":[165,205],"MBPP.":[167],"then":[169],"methodology":[172],"detection":[176],"experiment":[178],"with":[179],"open":[180],"source":[181],"such":[183],"CodeLLaMA":[185],"OpenAI's":[189],"GPT-4":[192,200],"models":[193],"using":[194],"one-shot":[195],"prompt.":[196],"find":[198],"that":[199],"performs":[201],"gives":[208],"comparable":[209],"results":[210],"fine-tuned":[213],"CodeBERT":[214],"baseline":[215],"MBPP":[217],"dataset.":[218],"Towards":[219],"end,":[221],"we":[222],"discuss":[223],"various":[224],"mitigation":[225],"strategies":[226],"conclude":[231],"work.":[233]},"counts_by_year":[{"year":2026,"cited_by_count":30},{"year":2025,"cited_by_count":29},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-09-13T00:00:00"}