{"id":"https://openalex.org/W4416874128","doi":"https://doi.org/10.1109/issrew67781.2025.00081","title":"An Evaluation Framework for Software Unit Tests Generated by Large Language Models","display_name":"An Evaluation Framework for Software Unit Tests Generated by Large Language Models","publication_year":2025,"publication_date":"2025-10-21","ids":{"openalex":"https://openalex.org/W4416874128","doi":"https://doi.org/10.1109/issrew67781.2025.00081"},"language":null,"primary_location":{"id":"doi:10.1109/issrew67781.2025.00081","is_oa":false,"landing_page_url":"https://doi.org/10.1109/issrew67781.2025.00081","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 36th International Symposium on Software Reliability Engineering Workshops (ISSREW)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103043980","display_name":"Steven Thanhtan Nguyen","orcid":"https://orcid.org/0000-0002-8789-3133"},"institutions":[{"id":"https://openalex.org/I26538001","display_name":"San Diego State University","ror":"https://ror.org/0264fdx42","country_code":"US","type":"education","lineage":["https://openalex.org/I26538001"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Steven Thanhtan Nguyen","raw_affiliation_strings":["San Diego State University,Department of Computer Science,San Diego,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"San Diego State University,Department of Computer Science,San Diego,USA","institution_ids":["https://openalex.org/I26538001"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Smita Bharath","orcid":null},"institutions":[{"id":"https://openalex.org/I26538001","display_name":"San Diego State University","ror":"https://ror.org/0264fdx42","country_code":"US","type":"education","lineage":["https://openalex.org/I26538001"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Smita Bharath","raw_affiliation_strings":["San Diego State University,Department of Computer Science,San Diego,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"San Diego State University,Department of Computer Science,San Diego,USA","institution_ids":["https://openalex.org/I26538001"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001028658","display_name":"Hajar Homayouni","orcid":"https://orcid.org/0000-0001-8898-9498"},"institutions":[{"id":"https://openalex.org/I26538001","display_name":"San Diego State University","ror":"https://ror.org/0264fdx42","country_code":"US","type":"education","lineage":["https://openalex.org/I26538001"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hajar Homayouni","raw_affiliation_strings":["San Diego State University,Department of Computer Science,San Diego,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"San Diego State University,Department of Computer Science,San Diego,USA","institution_ids":["https://openalex.org/I26538001"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I26538001"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.32763759,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"01","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9018999934196472,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9018999934196472,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.05000000074505806,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.021400000900030136,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/unit-testing","display_name":"Unit testing","score":0.8453999757766724},{"id":"https://openalex.org/keywords/maintainability","display_name":"Maintainability","score":0.6568999886512756},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.5134000182151794},{"id":"https://openalex.org/keywords/software-quality-assurance","display_name":"Software quality assurance","score":0.48420000076293945},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4810999929904938},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.476500004529953},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.453000009059906},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.44020000100135803},{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.40720000863075256},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4056999981403351}],"concepts":[{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.8453999757766724},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6801000237464905},{"id":"https://openalex.org/C160713754","wikidata":"https://www.wikidata.org/wiki/Q1389965","display_name":"Maintainability","level":2,"score":0.6568999886512756},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5699999928474426},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.5134000182151794},{"id":"https://openalex.org/C2776969324","wikidata":"https://www.wikidata.org/wiki/Q613918","display_name":"Software quality assurance","level":5,"score":0.48420000076293945},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4810999929904938},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.476500004529953},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.453000009059906},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.44020000100135803},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4230000078678131},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.40720000863075256},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4056999981403351},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.3993000090122223},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.398499995470047},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.37459999322891235},{"id":"https://openalex.org/C107683887","wikidata":"https://www.wikidata.org/wiki/Q782466","display_name":"Integration testing","level":3,"score":0.3614000082015991},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.35830000042915344},{"id":"https://openalex.org/C122637931","wikidata":"https://www.wikidata.org/wiki/Q118084","display_name":"Unit (ring theory)","level":2,"score":0.3221000134944916},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.3197999894618988},{"id":"https://openalex.org/C91071405","wikidata":"https://www.wikidata.org/wiki/Q1413145","display_name":"Program slicing","level":3,"score":0.31769999861717224},{"id":"https://openalex.org/C182122060","wikidata":"https://www.wikidata.org/wiki/Q6752328","display_name":"Manual testing","level":5,"score":0.31470000743865967},{"id":"https://openalex.org/C7435765","wikidata":"https://www.wikidata.org/wiki/Q7705776","display_name":"Test Management Approach","level":5,"score":0.30820000171661377},{"id":"https://openalex.org/C86469151","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Non-regression testing","level":5,"score":0.30790001153945923},{"id":"https://openalex.org/C165825675","wikidata":"https://www.wikidata.org/wiki/Q1399743","display_name":"Model-based testing","level":4,"score":0.3077000081539154},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3061000108718872},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.29589998722076416},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.29499998688697815},{"id":"https://openalex.org/C169168650","wikidata":"https://www.wikidata.org/wiki/Q1675637","display_name":"Keyword-driven testing","level":5,"score":0.2865000069141388},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.2840999960899353},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.2833999991416931},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2831000089645386},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.2808000147342682},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.27959999442100525},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.26510000228881836},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.2644999921321869},{"id":"https://openalex.org/C139143892","wikidata":"https://www.wikidata.org/wiki/Q7441615","display_name":"Search-based software engineering","level":5,"score":0.26080000400543213}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/issrew67781.2025.00081","is_oa":false,"landing_page_url":"https://doi.org/10.1109/issrew67781.2025.00081","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 36th International Symposium on Software Reliability Engineering Workshops (ISSREW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2060795628","https://openalex.org/W2135841285","https://openalex.org/W2145991289","https://openalex.org/W2154897437","https://openalex.org/W2587022626","https://openalex.org/W3011245529","https://openalex.org/W3135746251","https://openalex.org/W4236200536","https://openalex.org/W4379598302","https://openalex.org/W4387711873","https://openalex.org/W4389104713","https://openalex.org/W4399668074"],"related_works":[],"abstract_inverted_index":{"Faulty":[0],"software":[1,10,176],"unit":[2,67,126,160],"tests":[3,68,127],"can":[4],"lead":[5],"to":[6,125,172],"undetected":[7],"bugs,":[8],"undermining":[9],"quality,":[11],"reliability,":[12],"and":[13,35,40,43,75,95,104,109,116,133,157,169],"security.":[14],"While":[15],"prior":[16],"research":[17],"has":[18],"explored":[19],"automated":[20],"test":[21,34,145,161],"generation,":[22,162],"it":[23],"often":[24],"neglects":[25],"the":[26,63,184],"use":[27],"of":[28,65,153],"state-of-the-art":[29],"LLMs,":[30],"lacks":[31],"differentiation":[32],"between":[33],"program":[36],"faults,":[37],"overlooks":[38],"readability":[39],"maintainability":[41],"metrics,":[42],"rarely":[44],"leverages":[45],"prompt":[46,142],"engineering":[47,143],"for":[48,61,87,167],"optimization.":[49],"To":[50],"address":[51],"these":[52],"gaps,":[53],"this":[54,123],"paper":[55],"introduces":[56],"a":[57,83,150],"comprehensive":[58],"evaluation":[59],"framework":[60,78,124],"assessing":[62],"quality":[64],"LLM-generated":[66],"along":[69],"three":[70],"key":[71],"dimensions:":[72],"validity,":[73,88],"effectiveness,":[74],"readability.":[76],"The":[77],"combines":[79],"established":[80],"metrics":[81],"with":[82],"newly":[84],"proposed":[85],"metric":[86],"encompassing":[89],"Build":[90],"&":[91],"Successful":[92],"Execution":[93],"Rate":[94],"Unit":[96],"Test":[97],"Failure":[98],"Evaluation":[99],"(for":[100,107,119],"validity),":[101],"Line":[102],"Coverage":[103],"Mutation":[105],"Score":[106],"effectiveness),":[108],"Flesch-Kincaid":[110],"Grade":[111],"Level,":[112],"Gunning":[113],"Fog":[114],"Index,":[115],"Halstead":[117],"Metrics":[118],"readability).":[120],"We":[121],"apply":[122],"generated":[128],"by":[129],"two":[130],"leading":[131],"LLMs-GPT40":[132],"Llama":[134],"3-70B-8192-across":[135],"10":[136],"Java":[137],"programs,":[138],"also":[139],"examining":[140],"how":[141],"affects":[144],"quality.":[146],"Our":[147],"findings":[148],"provide":[149],"detailed":[151],"picture":[152],"current":[154],"LLMs'":[155],"capabilities":[156],"limitations":[158],"in":[159],"offe":[163],"ring":[164],"actionable":[165],"insights":[166],"researchers":[168],"practitioners":[170],"aiming":[171],"integrate":[173],"LLMs":[174],"into":[175],"testing":[177],"workflows.":[178],"Source":[179],"code":[180],"is":[181],"available":[182],"at":[183],"following":[185],"link":[186],"-":[187],"https://github.com/StevenTNguyen-Code/Quality-of-AI-Generated-Test-Cases":[188]},"counts_by_year":[],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-12-01T00:00:00"}