{"id":"https://openalex.org/W7128191878","doi":"https://doi.org/10.1145/3745764","title":"NLPerturbator: Studying the Robustness of Code LLMs to Natural Language Variations","display_name":"NLPerturbator: Studying the Robustness of Code LLMs to Natural Language Variations","publication_year":2025,"publication_date":"2025-07-08","ids":{"openalex":"https://openalex.org/W7128191878","doi":"https://doi.org/10.1145/3745764"},"language":"en","primary_location":{"id":"doi:10.1145/3745764","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3745764","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5121470880","display_name":"Junkai Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Junkai Chen","raw_affiliation_strings":["The State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, China and Singapore Management University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0009-0000-9945-7729","affiliations":[{"raw_affiliation_string":"The State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, China and Singapore Management University, Singapore, Singapore","institution_ids":["https://openalex.org/I79891267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004382658","display_name":"Zhenhao Li","orcid":"https://orcid.org/0000-0002-4909-1535"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zhenhao Li","raw_affiliation_strings":["York University, Toronto, Ontario, Canada"],"raw_orcid":"https://orcid.org/0000-0002-4909-1535","affiliations":[{"raw_affiliation_string":"York University, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xing Hu","orcid":"https://orcid.org/0000-0003-0093-3292"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xing Hu","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-0093-3292","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":null,"display_name":"Xin Xia","orcid":"https://orcid.org/0000-0002-6302-3256"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Xia","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-6302-3256","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.5175,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.94400171,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"35","issue":"4","first_page":"1","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3628000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3628000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.17520000040531158,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.07880000025033951,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7131999731063843},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.6561999917030334},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5396000146865845},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4309999942779541},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.4131999909877777}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7131999731063843},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7062000036239624},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.6561999917030334},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5396000146865845},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4309999942779541},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.4131999909877777},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.37790000438690186},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3659000098705292},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3273000121116638},{"id":"https://openalex.org/C49630185","wikidata":"https://www.wikidata.org/wiki/Q6980675","display_name":"Natural experiment","level":2,"score":0.31150001287460327},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.298799991607666},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2888000011444092},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2565000057220459}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3745764","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3745764","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5258774757385254,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G2982775919","display_name":null,"funder_award_id":"2024YFB4506400","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W3108032709","https://openalex.org/W4210499321"],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1,14,34,69,121,134],"models":[2],"achieve":[3],"promising":[4],"results":[5],"in":[6,36,57,73,123,211],"code":[7,113,175,191],"generation":[8,176,192],"based":[9,91,141],"on":[10,84,92,142,174],"a":[11,107,165,170,194],"given":[12,164],"natural":[13,33,68,120,133],"description.":[15],"They":[16],"have":[17,49],"been":[18],"integrated":[19],"into":[20],"open":[21],"source":[22],"projects":[23],"and":[24,80,95,135,146],"commercial":[25],"products":[26],"to":[27,43,55,110,117,208],"facilitate":[28],"daily":[29],"coding":[30],"activities.":[31],"The":[32],"description":[35],"the":[37,58,67,85,183,188,200,204,212,217,222],"prompt":[38],"is":[39],"crucial":[40],"for":[41],"LLMs":[42,52,88,114,207],"comprehend":[44],"users\u2019":[45],"requirements.":[46],"Prior":[47,82],"studies":[48,83],"uncovered":[50],"that":[51,63,182],"are":[53,115],"sensitive":[54],"changes":[56,62],"prompts,":[59,213],"including":[60],"slight":[61],"look":[64],"inconspicuous.":[65],"However,":[66],"descriptions":[70,122],"often":[71,90],"vary":[72],"real-world":[74,124,209],"scenarios":[75],"(e.g.,":[76],"different":[77],"formats,":[78],"grammar,":[79],"wording).":[81],"robustness":[86,205],"of":[87,119,130,132,138,161,167,172,190,202,206,219],"were":[89],"random":[93],"perturbations,":[94],"such":[96],"perturbations":[97,131,160],"may":[98],"not":[99],"actually":[100],"happen.":[101],"In":[102],"this":[103],"article,":[104],"we":[105,180],"conduct":[106],"comprehensive":[108],"study":[109,198],"investigate":[111],"how":[112],"robust":[116],"variations":[118,210],"scenarios.":[125],"We":[126,151],"summarize":[127],"18":[128],"categories":[129,140],"three":[136],"combinations":[137],"co-occurred":[139],"our":[143],"literature":[144],"review":[145],"online":[147],"survey":[148],"with":[149],"practitioners.":[150],"propose":[152],"an":[153],"automated":[154],"framework,":[155],"NLPerturbator,":[156],"which":[157],"can":[158,186],"perform":[159],"each":[162],"category":[163],"set":[166],"prompts.":[168,223],"Through":[169],"series":[171],"experiments":[173],"using":[177],"sevencode":[178],"LLMs,":[179],"find":[181],"perturbed":[184],"prompts":[185],"decrease":[187],"performance":[189],"by":[193],"considerable":[195],"margin.":[196],"Our":[197],"highlights":[199],"importance":[201],"enhancing":[203],"as":[214,216],"well":[215],"essentiality":[218],"attentively":[220],"constructing":[221]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-02-08T00:00:00"}