{"id":"https://openalex.org/W4417036936","doi":"https://doi.org/10.5753/sbqs.2025.13792","title":"Evaluating LLM-Generated Unit Tests with Mutation Testing: ChatGPT vs DeepSeek","display_name":"Evaluating LLM-Generated Unit Tests with Mutation Testing: ChatGPT vs DeepSeek","publication_year":2025,"publication_date":"2025-11-04","ids":{"openalex":"https://openalex.org/W4417036936","doi":"https://doi.org/10.5753/sbqs.2025.13792"},"language":null,"primary_location":{"id":"doi:10.5753/sbqs.2025.13792","is_oa":true,"landing_page_url":"https://doi.org/10.5753/sbqs.2025.13792","pdf_url":"https://sol.sbc.org.br/index.php/sbqs/article/download/39001/38773","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Anais do XXIV Simp\u00f3sio Brasileiro de Qualidade de Software (SBQS 2025)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://sol.sbc.org.br/index.php/sbqs/article/download/39001/38773","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038604878","display_name":"Pedro Cabral","orcid":"https://orcid.org/0000-0001-8622-6008"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pedro Fernando Marinho Cabral","raw_affiliation_strings":["UFPA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFPA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114622196","display_name":"Jo\u00e3o Pedro Souza Arruda","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jo\u00e3o Pedro Souza Arruda","raw_affiliation_strings":["UFPA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFPA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064508238","display_name":"Cleidson R. B. de Souza","orcid":"https://orcid.org/0000-0003-3240-3122"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cleidson Ronald Botelho de Souza","raw_affiliation_strings":["UFPA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFPA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076087337","display_name":"Victor Hugo Santiago C. Pinto","orcid":"https://orcid.org/0000-0001-8562-6384"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Victor Hugo Santiago Costa Pinto","raw_affiliation_strings":["UFPA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFPA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"120","last_page":"130"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.965499997138977,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.965499997138977,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.011699999682605267,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.00279999990016222,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cyclomatic-complexity","display_name":"Cyclomatic complexity","score":0.7533000111579895},{"id":"https://openalex.org/keywords/unit-testing","display_name":"Unit testing","score":0.6521999835968018},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.5734999775886536},{"id":"https://openalex.org/keywords/mutation-testing","display_name":"Mutation testing","score":0.5347999930381775},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.4916999936103821},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.4571000039577484},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.4287000000476837},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.4090000092983246}],"concepts":[{"id":"https://openalex.org/C187303228","wikidata":"https://www.wikidata.org/wiki/Q867330","display_name":"Cyclomatic complexity","level":3,"score":0.7533000111579895},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.6521999835968018},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6335999965667725},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.5734999775886536},{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.5347999930381775},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.4916999936103821},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.4571000039577484},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.43860000371932983},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.4287000000476837},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.4090000092983246},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4011000096797943},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4007999897003174},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.38760000467300415},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3756999969482422},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.367900013923645},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.36090001463890076},{"id":"https://openalex.org/C8880873","wikidata":"https://www.wikidata.org/wiki/Q187787","display_name":"Genetic algorithm","level":2,"score":0.35260000824928284},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34360000491142273},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3418000042438507},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3301999866962433},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.328900009393692},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.31869998574256897},{"id":"https://openalex.org/C122637931","wikidata":"https://www.wikidata.org/wiki/Q118084","display_name":"Unit (ring theory)","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.265500009059906},{"id":"https://openalex.org/C2780673598","wikidata":"https://www.wikidata.org/wiki/Q285223","display_name":"Genetic testing","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C2984968299","wikidata":"https://www.wikidata.org/wiki/Q1077784","display_name":"Software tool","level":3,"score":0.25949999690055847},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2554999887943268}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5753/sbqs.2025.13792","is_oa":true,"landing_page_url":"https://doi.org/10.5753/sbqs.2025.13792","pdf_url":"https://sol.sbc.org.br/index.php/sbqs/article/download/39001/38773","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Anais do XXIV Simp\u00f3sio Brasileiro de Qualidade de Software (SBQS 2025)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.5753/sbqs.2025.13792","is_oa":true,"landing_page_url":"https://doi.org/10.5753/sbqs.2025.13792","pdf_url":"https://sol.sbc.org.br/index.php/sbqs/article/download/39001/38773","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Anais do XXIV Simp\u00f3sio Brasileiro de Qualidade de Software (SBQS 2025)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322025","display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","ror":"https://ror.org/03swz6y49"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4417036936.pdf","grobid_xml":"https://content.openalex.org/works/W4417036936.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W1964962870","https://openalex.org/W2049695835","https://openalex.org/W2135841285","https://openalex.org/W2784445699","https://openalex.org/W2951506308","https://openalex.org/W4298110250","https://openalex.org/W4308627320","https://openalex.org/W4312438588","https://openalex.org/W4384345639","https://openalex.org/W4385675875","https://openalex.org/W4391021472","https://openalex.org/W4391835303","https://openalex.org/W4391871252","https://openalex.org/W4391923915","https://openalex.org/W4391974543","https://openalex.org/W4402275888","https://openalex.org/W4402442864","https://openalex.org/W4402665833","https://openalex.org/W4403536268","https://openalex.org/W4403536388","https://openalex.org/W4411552119","https://openalex.org/W4411552855"],"related_works":[],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,11,16,26,35,55,151],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"have":[7],"driven":[8],"significant":[9,172],"progress":[10],"automating":[12],"software":[13],"testing,":[14],"particularly":[15],"generating":[17,56,107],"unit":[18,57],"tests.":[19],"However,":[20],"the":[21,36,47,64,79,92,190,201],"effectiveness":[22,150],"of":[23,49,70,81,94,139,168,189,203],"these":[24],"models":[25],"detecting":[27],"real":[28],"defects":[29],"through":[30],"mutation":[31],"testing":[32],"remains":[33],"underexplored":[34],"literature.":[37],"This":[38,192],"study":[39,193],"aims":[40],"to":[41,77,83,101,186],"address":[42],"this":[43],"gap":[44],"by":[45,159],"evaluating":[46],"performance":[48],"ChatGPT":[50,155],"(GPT-4o)":[51],"and":[52,87,97,121,130,149,179,197],"DeepSeek":[53,145],"V3":[54],"tests":[58,108],"for":[59,118,164,205,211],"six":[60],"Java":[61],"classes":[62],"from":[63],"Defects4J":[65],"dataset,":[66],"covering":[67],"different":[68],"levels":[69],"cyclomatic":[71,177],"complexity.":[72],"The":[73,104],"main":[74],"objective":[75],"is":[76],"investigate":[78],"ability":[80],"LLMs":[82,204],"maximize":[84],"mutant":[85],"coverage":[86],"elimination,":[88],"while":[89],"also":[90],"analyzing":[91],"impact":[93],"code":[95],"complexity":[96,178],"semantic":[98,187],"factors":[99],"related":[100],"execution":[102],"failures.":[103,141],"methodology":[105],"involved":[106],"via":[109],"structured":[110],"prompts,":[111],"executing":[112],"them":[113],"5":[114],"times":[115],"per":[116],"class":[117],"both":[119,195],"models,":[120],"performing":[122],"quantitative":[123,196],"analysis":[124,138],"based":[125],"on":[126,200],"Mutation":[127,131],"Coverage":[128],"(MC)":[129],"Score":[132],"(MS),":[133],"as":[134,136],"well":[135],"qualitative":[137,198],"runtime":[140],"Results":[142],"indicate":[143],"that":[144],"exhibits":[146],"greater":[147],"stability":[148],"eliminating":[152],"mutants,":[153],"whereas":[154],"demonstrates":[156],"broader":[157],"applicability":[158],"producing":[160],"valid":[161],"test":[162,207,214],"suites":[163],"a":[165],"wider":[166],"range":[167],"classes.":[169],"Moreover,":[170],"no":[171],"correlation":[173],"was":[174],"found":[175],"between":[176],"compilation":[180],"success,":[181],"with":[182],"failures":[183],"primarily":[184],"linked":[185],"limitations":[188],"models.":[191],"presents":[194],"evidence":[199],"application":[202],"automated":[206],"generation,":[208],"offering":[209],"insights":[210],"future":[212],"AI-driven":[213],"engineering":[215],"strategies.":[216]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-12-05T00:00:00"}