{"id":"https://openalex.org/W7125924498","doi":"https://doi.org/10.1109/smc58881.2025.11343558","title":"SPRoC: Semantics-Preserving Mutations for Robustness Evaluation of Code Generation Large Language Models","display_name":"SPRoC: Semantics-Preserving Mutations for Robustness Evaluation of Code Generation Large Language Models","publication_year":2025,"publication_date":"2025-10-05","ids":{"openalex":"https://openalex.org/W7125924498","doi":"https://doi.org/10.1109/smc58881.2025.11343558"},"language":null,"primary_location":{"id":"doi:10.1109/smc58881.2025.11343558","is_oa":false,"landing_page_url":"https://doi.org/10.1109/smc58881.2025.11343558","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Systems, Man, and Cybernetics (SMC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001460324","display_name":"Qiancheng Shi","orcid":null},"institutions":[{"id":"https://openalex.org/I78675632","display_name":"Beijing Information Science & Technology University","ror":"https://ror.org/04xnqep60","country_code":"CN","type":"education","lineage":["https://openalex.org/I78675632"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Qiancheng Shi","raw_affiliation_strings":["Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192"],"affiliations":[{"raw_affiliation_string":"Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192","institution_ids":["https://openalex.org/I78675632"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104384519","display_name":"Qihong Han","orcid":null},"institutions":[{"id":"https://openalex.org/I78675632","display_name":"Beijing Information Science & Technology University","ror":"https://ror.org/04xnqep60","country_code":"CN","type":"education","lineage":["https://openalex.org/I78675632"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qihong Han","raw_affiliation_strings":["Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192"],"affiliations":[{"raw_affiliation_string":"Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192","institution_ids":["https://openalex.org/I78675632"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124080776","display_name":"Zhanqi Cui","orcid":null},"institutions":[{"id":"https://openalex.org/I78675632","display_name":"Beijing Information Science & Technology University","ror":"https://ror.org/04xnqep60","country_code":"CN","type":"education","lineage":["https://openalex.org/I78675632"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhanqi Cui","raw_affiliation_strings":["Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192"],"affiliations":[{"raw_affiliation_string":"Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192","institution_ids":["https://openalex.org/I78675632"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101859130","display_name":"Zheng Zeng","orcid":"https://orcid.org/0000-0001-9025-9427"},"institutions":[{"id":"https://openalex.org/I78675632","display_name":"Beijing Information Science & Technology University","ror":"https://ror.org/04xnqep60","country_code":"CN","type":"education","lineage":["https://openalex.org/I78675632"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheng Zeng","raw_affiliation_strings":["Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192"],"affiliations":[{"raw_affiliation_string":"Beijing Information Science and Technology University,College of Computer Science,Beijing,China,100192","institution_ids":["https://openalex.org/I78675632"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5001460324"],"corresponding_institution_ids":["https://openalex.org/I78675632"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.83882177,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2218","last_page":"2223"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.225600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.225600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.17170000076293945,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.15719999372959137,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7821999788284302},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6751000285148621},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.48089998960494995},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.3538999855518341},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3246999979019165},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.2992999851703644}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7821999788284302},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7815999984741211},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6751000285148621},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.48089998960494995},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3831999897956848},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3788999915122986},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.3538999855518341},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3488999903202057},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3246999979019165},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.29899999499320984},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.29440000653266907},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.29170000553131104},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2913999855518341},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.2759000062942505}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/smc58881.2025.11343558","is_oa":false,"landing_page_url":"https://doi.org/10.1109/smc58881.2025.11343558","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Systems, Man, and Cybernetics (SMC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320326666","display_name":"Beijing Information Science and Technology University","ror":"https://ror.org/04xnqep60"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2768572539","https://openalex.org/W2955426500","https://openalex.org/W4367061230","https://openalex.org/W4384026520","https://openalex.org/W4384345698","https://openalex.org/W4385562549","https://openalex.org/W4388412504","https://openalex.org/W4389524484","https://openalex.org/W4400484693"],"related_works":[],"abstract_inverted_index":{"With":[0],"the":[1,60,121,128,149,174,183],"widespread":[2],"use":[3],"of":[4,63,75,85,123,131,140,195],"large":[5],"language":[6],"models":[7],"(LLMs)":[8],"in":[9,193],"code":[10,72,132],"generation,":[11],"their":[12,68],"capabilities":[13],"continues":[14],"enhanced.":[15],"However,":[16],"LLMs":[17,141],"still":[18,51],"exhibit":[19],"instability":[20],"when":[21],"faced":[22],"with":[23,152,178],"minor":[24],"input":[25,143],"prompt":[26,35,47,95],"variations,":[27],"which":[28],"presents":[29],"challenges":[30],"for":[31,90],"practical":[32],"deployment.":[33],"Existing":[34],"mutation":[36,136],"methods,":[37],"such":[38],"as":[39],"random":[40],"insertions,":[41],"deletions,":[42],"or":[43],"replacements":[44],"without":[45],"understanding":[46],"semantics":[48],"and":[49,134,161,200,205],"structure,":[50],"have":[52],"some":[53],"limitations.":[54],"These":[55,113],"methods":[56],"fail":[57],"to":[58,70,119,137,142,163],"capture":[59],"diverse":[61,111],"ways":[62],"real":[64],"users":[65],"express,":[66],"limiting":[67],"ability":[69],"assess":[71,138],"generation":[73],"robustness":[74,93,139],"LLMs.":[76],"To":[77],"address":[78],"this,":[79],"we":[80],"propose":[81],"SPRoC":[82,101,126,167,172,189],"(Semantics-Preserving":[83],"Robustness":[84],"Code":[86],"generation),":[87],"a":[88,98,116],"method":[89],"evaluating":[91],"LLM":[92],"through":[94],"mutation.":[96],"Using":[97],"BERT-based":[99],"model,":[100],"generates":[102],"mutated":[103],"prompts":[104,114],"that":[105,171],"maintain":[106],"semantic":[107,180],"consistency":[108],"but":[109],"offer":[110],"expressions.":[112],"create":[115],"new":[117],"dataset":[118,151],"verify":[120],"functionality":[122],"LLM-generated":[124],"code.":[125],"compares":[127],"functional":[129],"correctness":[130],"before":[133],"after":[135],"variations.":[144],"We":[145],"conduct":[146],"experiments":[147],"on":[148],"HumanEval":[150],"several":[153],"mainstream":[154],"LLMs,":[155],"including":[156],"ChatGPT,":[157],"DeepSeek,":[158],"Claude,":[159],"ERNIE,":[160],"Qwen,":[162],"evaluate":[164],"performance":[165,192],"under":[166],"mutations.":[168],"Results":[169],"show":[170],"reduces":[173],"models\u2019":[175],"Pass@k":[176],"scores":[177],"minimal":[179],"changes,":[181],"outperforming":[182],"baseline":[184],"Radamsa":[185],"method.":[186],"In":[187],"addition,":[188],"achieves":[190],"better":[191],"terms":[194],"similarity":[196],"metrics":[197],"like":[198],"BLEU":[199],"BERTScore,":[201],"improving":[202],"by":[203],"12.96%":[204],"1.83%,":[206],"respectively.":[207]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2026-01-29T00:00:00"}