{"id":"https://openalex.org/W7162124473","doi":"https://doi.org/10.48550/arxiv.2605.22175","title":"SWE-Mutation: Can LLMs Generate Reliable Test Suites in Software Engineering?","display_name":"SWE-Mutation: Can LLMs Generate Reliable Test Suites in Software Engineering?","publication_year":2026,"publication_date":"2026-05-21","ids":{"openalex":"https://openalex.org/W7162124473","doi":"https://doi.org/10.48550/arxiv.2605.22175"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.22175","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22175","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.22175","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136799167","display_name":"Yuxuan Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Yuxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073449521","display_name":"Yuze Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yuze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100367924","display_name":"Yufeng Wang","orcid":"https://orcid.org/0000-0003-0448-325X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yufeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101498554","display_name":"Yao Du","orcid":"https://orcid.org/0000-0003-2914-2283"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136776836","display_name":"Zhiyuan Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Zhiyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136743706","display_name":"Jinbo Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jinbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136790465","display_name":"Mengdi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Mengdi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136774410","display_name":"Kai Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136779547","display_name":"Zhenya Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zhenya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8693000078201294,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8693000078201294,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.09799999743700027,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.007400000002235174,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7973999977111816},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6881999969482422},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.667900025844574},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6079999804496765},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5435000061988831},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.43309998512268066},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.41519999504089355},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.38109999895095825}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7973999977111816},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6881999969482422},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.667900025844574},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6425999999046326},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6079999804496765},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5435000061988831},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.43309998512268066},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41589999198913574},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.41519999504089355},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.38109999895095825},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33809998631477356},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3212999999523163},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.29679998755455017},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.288100004196167},{"id":"https://openalex.org/C109747225","wikidata":"https://www.wikidata.org/wiki/Q815758","display_name":"Scarcity","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.2775999903678894},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.2750000059604645},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.26350000500679016},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.26109999418258667},{"id":"https://openalex.org/C78766204","wikidata":"https://www.wikidata.org/wiki/Q555032","display_name":"Multi-core processor","level":2,"score":0.2581999897956848}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.22175","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22175","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.22175","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.22175","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.7155623435974121,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"software":[1],"engineering":[2],"capabilities":[3],"has":[4],"become":[5],"a":[6,92,103,156],"core":[7],"component":[8],"of":[9,27,34,65,145,182,212],"modern":[10],"large":[11],"language":[12],"models":[13],"(LLMs);":[14],"however,":[15],"the":[16,25,32,60,124,180,210],"key":[17],"bottleneck":[18],"hindering":[19],"further":[20,131],"scaling":[21],"lies":[22],"not":[23],"in":[24,31,54,209],"scarcity":[26],"high-quality":[28,35,67,97],"solutions,":[29],"but":[30],"lack":[33,87],"test":[36,68,98,108,113,125,220],"suites.":[37,109,221],"Test":[38],"suites":[39,69,114,126],"are":[40],"indispensable":[41],"both":[42],"for":[43,49,105,137],"synthesizing":[44],"program":[45],"repair":[46],"trajectories":[47],"and":[48,63,86,127,154,175,218],"providing":[50],"precise":[51],"feedback":[52],"signals":[53],"reinforcement":[55],"learning.":[56],"Unfortunately,":[57],"due":[58],"to":[59,74,83,122,198,201,215],"high":[61],"cost":[62],"difficulty":[64],"annotation,":[66],"have":[70],"long":[71],"been":[72],"hard":[73],"obtain,":[75],"while":[76],"those":[77],"automatically":[78,138],"generated":[79],"by":[80,115],"LLMs":[81,166,214],"tend":[82],"be":[84],"superficial":[85],"sufficient":[88],"discriminative":[89,219],"power.":[90],"As":[91],"first":[93],"step":[94],"toward":[95],"constructing":[96],"suites,":[99],"we":[100],"introduce":[101],"SWE-Mutation,":[102],"benchmark":[104,111,143],"evaluating":[106],"LLM-generated":[107],"The":[110],"characterizes":[112],"introducing":[116],"systematically":[117],"mutated":[118,147],"solutions":[119],"that":[120,168],"attempt":[121],"``fool''":[123],"pass":[128],"validation.":[129],"We":[130],"propose":[132],"an":[133],"agentic,":[134],"language-agnostic":[135],"framework":[136],"generating":[139],"complex":[140],"mutants.":[141],"Our":[142],"consists":[144],"2,636":[146],"variants":[148],"derived":[149],"from":[150,196],"800":[151],"original":[152],"instances":[153],"includes":[155],"multilingual":[157],"subset":[158],"spanning":[159],"nine":[160],"programming":[161],"languages.":[162],"Experiments":[163],"on":[164],"seven":[165],"reveal":[167],"even":[169],"DeepSeek-V3.1":[170],"achieves":[171],"only":[172],"10.20%":[173],"verification":[174],"36.15%":[176],"detection":[177,194],"rates,":[178],"highlighting":[179],"inadequacy":[181],"current":[183,213],"LLMs.":[184],"Additionally,":[185],"our":[186],"agentic":[187],"mutation":[188],"strategy":[189],"enhances":[190],"realism,":[191],"reducing":[192],"average":[193],"rates":[195],"71.04%":[197],"39.81%":[199],"compared":[200],"conventional":[202],"methods.":[203],"These":[204],"findings":[205],"expose":[206],"persistent":[207],"deficiencies":[208],"ability":[211],"generate":[216],"reliable":[217]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-23T00:00:00"}