{"id":"https://openalex.org/W4412420766","doi":"https://doi.org/10.1145/3748647","title":"Pitfalls in Language Models for Code Intelligence: A Taxonomy and Survey","display_name":"Pitfalls in Language Models for Code Intelligence: A Taxonomy and Survey","publication_year":2025,"publication_date":"2025-07-15","ids":{"openalex":"https://openalex.org/W4412420766","doi":"https://doi.org/10.1145/3748647"},"language":"en","primary_location":{"id":"doi:10.1145/3748647","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3748647","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111136936","display_name":"Xinyu She","orcid":null},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinyu She","raw_affiliation_strings":["Huazhong University of Science and Technology, Wuhan, China","Huazhong University of Science and Technology, China"],"raw_orcid":"https://orcid.org/0009-0001-2988-7042","affiliations":[{"raw_affiliation_string":"Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]},{"raw_affiliation_string":"Huazhong University of Science and Technology, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100320074","display_name":"Yue Liu","orcid":"https://orcid.org/0000-0002-5593-5917"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yue Liu","raw_affiliation_strings":["Monash University, Melbourne, Australia","Monash University, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5593-5917","affiliations":[{"raw_affiliation_string":"Monash University, Melbourne, Australia","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"Monash University, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023970004","display_name":"Yanjie Zhao","orcid":"https://orcid.org/0000-0001-8793-5367"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanjie Zhao","raw_affiliation_strings":["Huazhong University of Science and Technology, Wuhan, China","Huazhong University of Science and Technology, China"],"raw_orcid":"https://orcid.org/0000-0001-8793-5367","affiliations":[{"raw_affiliation_string":"Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]},{"raw_affiliation_string":"Huazhong University of Science and Technology, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101935489","display_name":"Yiling He","orcid":"https://orcid.org/0000-0002-5977-1489"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiling He","raw_affiliation_strings":["Zhejiang University, Hangzhou, China","Zhejiang University, China"],"raw_orcid":"https://orcid.org/0000-0002-5977-1489","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]},{"raw_affiliation_string":"Zhejiang University, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106407830","display_name":"Li Li","orcid":"https://orcid.org/0000-0003-2990-1614"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Li","raw_affiliation_strings":["Beihang University, Beijing, China","Beihang University, China"],"raw_orcid":"https://orcid.org/0000-0003-2990-1614","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"Beihang University, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081449581","display_name":"Chakkrit Tantithamthavorn","orcid":"https://orcid.org/0000-0002-5516-9984"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Chakkrit Tantithamthavorn","raw_affiliation_strings":["Information Technology, Monash University, Clayton, Australia","Monash University, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5516-9984","affiliations":[{"raw_affiliation_string":"Information Technology, Monash University, Clayton, Australia","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"Monash University, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043524348","display_name":"Zhan Qin","orcid":"https://orcid.org/0000-0001-7872-6969"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhan Qin","raw_affiliation_strings":["Zhejiang University, Hangzhou, China","Zhejiang University, China"],"raw_orcid":"https://orcid.org/0000-0001-7872-6969","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]},{"raw_affiliation_string":"Zhejiang University, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5115695530","display_name":"Haoyu Wang","orcid":"https://orcid.org/0000-0003-1100-8633"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoyu Wang","raw_affiliation_strings":["Huazhong University of Science and Technology, Wuhan, China","Huazhong University of Science and Technology, China"],"raw_orcid":"https://orcid.org/0000-0003-1100-8633","affiliations":[{"raw_affiliation_string":"Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]},{"raw_affiliation_string":"Huazhong University of Science and Technology, China","institution_ids":["https://openalex.org/I47720641"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.08,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.96123625,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"35","issue":"3","first_page":"1","last_page":"42"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8306677341461182},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.6398881673812866},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.45080631971359253},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4375850260257721},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36118409037590027},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32199713587760925}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8306677341461182},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.6398881673812866},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.45080631971359253},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4375850260257721},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36118409037590027},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32199713587760925},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3748647","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3748647","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":126,"referenced_works":["https://openalex.org/W1985229168","https://openalex.org/W2133933998","https://openalex.org/W2242083635","https://openalex.org/W2408181256","https://openalex.org/W2508327075","https://openalex.org/W2749460454","https://openalex.org/W2765527199","https://openalex.org/W2787225861","https://openalex.org/W2808113972","https://openalex.org/W2883754520","https://openalex.org/W2888312537","https://openalex.org/W2888557792","https://openalex.org/W2907083496","https://openalex.org/W2912899164","https://openalex.org/W2951914319","https://openalex.org/W2954276596","https://openalex.org/W2964315653","https://openalex.org/W2964322208","https://openalex.org/W2964325845","https://openalex.org/W2997451752","https://openalex.org/W3000315285","https://openalex.org/W3004658838","https://openalex.org/W3007007518","https://openalex.org/W3011013751","https://openalex.org/W3011632945","https://openalex.org/W3014780393","https://openalex.org/W3023529621","https://openalex.org/W3081454995","https://openalex.org/W3084812981","https://openalex.org/W3089189066","https://openalex.org/W3091588759","https://openalex.org/W3093464584","https://openalex.org/W3103802018","https://openalex.org/W3105867435","https://openalex.org/W3108032709","https://openalex.org/W3131641316","https://openalex.org/W3134763859","https://openalex.org/W3138904555","https://openalex.org/W3156669901","https://openalex.org/W3157441214","https://openalex.org/W3161746459","https://openalex.org/W3163206498","https://openalex.org/W3163672235","https://openalex.org/W3174414731","https://openalex.org/W3174750614","https://openalex.org/W3177116043","https://openalex.org/W3178061567","https://openalex.org/W3187467055","https://openalex.org/W3194346579","https://openalex.org/W3194549834","https://openalex.org/W3195703954","https://openalex.org/W3195727321","https://openalex.org/W3195958055","https://openalex.org/W3199693282","https://openalex.org/W3202183072","https://openalex.org/W3210860486","https://openalex.org/W3211225893","https://openalex.org/W4200633062","https://openalex.org/W4205601502","https://openalex.org/W4206238894","https://openalex.org/W4210440021","https://openalex.org/W4210499321","https://openalex.org/W4210550462","https://openalex.org/W4210772589","https://openalex.org/W4210918116","https://openalex.org/W4220722393","https://openalex.org/W4220914596","https://openalex.org/W4221157342","https://openalex.org/W4225640545","https://openalex.org/W4226294137","https://openalex.org/W4226463316","https://openalex.org/W4281763794","https://openalex.org/W4283751459","https://openalex.org/W4284667247","https://openalex.org/W4284676027","https://openalex.org/W4284710241","https://openalex.org/W4285141508","https://openalex.org/W4285248702","https://openalex.org/W4285490465","https://openalex.org/W4285678688","https://openalex.org/W4285798540","https://openalex.org/W4287673430","https://openalex.org/W4288057765","https://openalex.org/W4288077069","https://openalex.org/W4288758078","https://openalex.org/W4301880089","https://openalex.org/W4308627220","https://openalex.org/W4308641647","https://openalex.org/W4308641648","https://openalex.org/W4308643013","https://openalex.org/W4308648312","https://openalex.org/W4312438588","https://openalex.org/W4312490312","https://openalex.org/W4312971267","https://openalex.org/W4313547593","https://openalex.org/W4313563755","https://openalex.org/W4319451764","https://openalex.org/W4319663674","https://openalex.org/W4367059303","https://openalex.org/W4367059627","https://openalex.org/W4367672983","https://openalex.org/W4380551114","https://openalex.org/W4384155620","https://openalex.org/W4384304728","https://openalex.org/W4384304811","https://openalex.org/W4384345640","https://openalex.org/W4384345684","https://openalex.org/W4384345698","https://openalex.org/W4384345708","https://openalex.org/W4384345728","https://openalex.org/W4384345745","https://openalex.org/W4385187279","https://openalex.org/W4385884967","https://openalex.org/W4388229767","https://openalex.org/W4391174792","https://openalex.org/W4391558518","https://openalex.org/W4391696889","https://openalex.org/W4391974543","https://openalex.org/W4394744221","https://openalex.org/W4395479870","https://openalex.org/W4400582422","https://openalex.org/W4402443087","https://openalex.org/W4402457600","https://openalex.org/W4402665833","https://openalex.org/W4403536562","https://openalex.org/W4403536900"],"related_works":["https://openalex.org/W3156937073","https://openalex.org/W2530757009","https://openalex.org/W3195027199","https://openalex.org/W4231937131","https://openalex.org/W3188962172","https://openalex.org/W323219885","https://openalex.org/W274175260","https://openalex.org/W2063928587","https://openalex.org/W2772917594","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Modern":[0],"Language":[1],"Models":[2],"(LMs)":[3],"have":[4,123],"been":[5,124],"successfully":[6],"employed":[7],"in":[8,19,61,114,137,207],"source":[9],"code":[10,24,41,94],"generation":[11],"and":[12,31,54,59,84,140,152,175,179,183,185,198,203,209],"understanding,":[13],"leading":[14],"to":[15,46,87,93,109,145,192],"a":[16,71,98,133,142,162,194],"significant":[17],"increase":[18],"research":[20,101,139],"focused":[21],"on":[22,97],"learning-based":[23],"intelligence,":[25],"such":[26],"as":[27],"automated":[28],"bug":[29],"repair":[30],"test":[32],"case":[33],"generation.":[34],"Despite":[35],"their":[36,57,81,201],"great":[37],"potential,":[38],"LMs":[39,91],"for":[40,70,157,196],"intelligence":[42],"(LM4Code)":[43],"are":[44],"susceptible":[45],"potential":[47],"pitfalls":[48,112,136,156,168],",":[49],"which":[50],"hinder":[51],"realistic":[52],"performance":[53,181],"further":[55],"impact":[56],"reliability":[58],"applicability":[60],"real-world":[62],"deployment":[63,184],".":[64],"Such":[65],"challenges":[66,153],"drive":[67],"the":[68,111,147],"need":[69],"comprehensive":[72,163],"understanding\u2014not":[73],"just":[74],"identifying":[75],"these":[76,129],"issues":[77],"but":[78],"delving":[79],"into":[80],"possible":[82],"implications":[83],"existing":[85],"solutions":[86],"build":[88],"more":[89],"reliable":[90,208],"tailored":[92],"intelligence.":[95],"Based":[96],"well-defined":[99],"systematic":[100,143],"approach,":[102],"we":[103,131,190],"conducted":[104,141],"an":[105],"extensive":[106],"literature":[107],"review":[108],"uncover":[110],"inherent":[113],"LM4Code.":[115],"Finally,":[116],"121":[117],"primary":[118],"studies":[119],"from":[120],"top-tier":[121],"venues":[122],"identified.":[125],"After":[126],"carefully":[127],"examining":[128],"studies,":[130],"designed":[132],"taxonomy":[134],"of":[135,154,205],"LM4Code":[138,158,206],"study":[144],"summarize":[146],"issues,":[148],"current":[149],"solutions,":[150],"implications,":[151],"different":[155],"systems.":[159],"We":[160],"developed":[161],"classification":[164],"scheme":[165],"that":[166],"dissects":[167],"across":[169],"four":[170],"crucial":[171],"aspects:":[172],"data":[173],"collection":[174],"labeling,":[176],"system":[177],"design":[178],"learning,":[180],"evaluation,":[182],"maintenance.":[186],"Through":[187],"this":[188],"study,":[189],"aim":[191],"provide":[193],"roadmap":[195],"researchers":[197],"practitioners,":[199],"facilitating":[200],"understanding":[202],"utilization":[204],"trustworthy":[210],"ways.":[211]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}