{"id":"https://openalex.org/W4408290845","doi":"https://doi.org/10.3390/software4010005","title":"A Systematic Approach for Assessing Large Language Models\u2019 Test Case Generation Capability","display_name":"A Systematic Approach for Assessing Large Language Models\u2019 Test Case Generation Capability","publication_year":2025,"publication_date":"2025-03-10","ids":{"openalex":"https://openalex.org/W4408290845","doi":"https://doi.org/10.3390/software4010005"},"language":"en","primary_location":{"id":"doi:10.3390/software4010005","is_oa":true,"landing_page_url":"https://doi.org/10.3390/software4010005","pdf_url":"https://www.mdpi.com/2674-113X/4/1/5/pdf?version=1741599844","source":{"id":"https://openalex.org/S4210183365","display_name":"Software","issn_l":"2674-113X","issn":["2674-113X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://www.mdpi.com/2674-113X/4/1/5/pdf?version=1741599844","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003606295","display_name":"Hung\u2013Fu Chang","orcid":"https://orcid.org/0000-0003-2913-4419"},"institutions":[{"id":"https://openalex.org/I135191193","display_name":"University of Indianapolis","ror":"https://ror.org/052133d12","country_code":"US","type":"education","lineage":["https://openalex.org/I135191193"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Hung-Fu Chang","raw_affiliation_strings":["R. B. Annis School of Engineering, University of Indianapolis, Indianapolis, IN 46227, USA"],"affiliations":[{"raw_affiliation_string":"R. B. Annis School of Engineering, University of Indianapolis, Indianapolis, IN 46227, USA","institution_ids":["https://openalex.org/I135191193"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109405225","display_name":"Mohammad Shokrolah Shirazi","orcid":"https://orcid.org/0000-0003-1775-0257"},"institutions":[{"id":"https://openalex.org/I917324061","display_name":"Marian University - Indiana","ror":"https://ror.org/00p4ywg82","country_code":"US","type":"education","lineage":["https://openalex.org/I917324061"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammad Shokrolah Shirazi","raw_affiliation_strings":["E. S. Witchger School of Engineering, Marian University, Indianapolis, IN 46222, USA"],"affiliations":[{"raw_affiliation_string":"E. S. Witchger School of Engineering, Marian University, Indianapolis, IN 46222, USA","institution_ids":["https://openalex.org/I917324061"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5003606295"],"corresponding_institution_ids":["https://openalex.org/I135191193"],"apc_list":null,"apc_paid":null,"fwci":11.8428,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.98233469,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"4","issue":"1","first_page":"5","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.594751238822937},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.47281160950660706},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.38109833002090454},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.19988909363746643},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.09186670184135437}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.594751238822937},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.47281160950660706},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.38109833002090454},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.19988909363746643},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.09186670184135437},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/software4010005","is_oa":true,"landing_page_url":"https://doi.org/10.3390/software4010005","pdf_url":"https://www.mdpi.com/2674-113X/4/1/5/pdf?version=1741599844","source":{"id":"https://openalex.org/S4210183365","display_name":"Software","issn_l":"2674-113X","issn":["2674-113X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:d27c675f68824623a1f2956177794e31","is_oa":true,"landing_page_url":"https://doaj.org/article/d27c675f68824623a1f2956177794e31","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Software, Vol 4, Iss 1, p 5 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/software4010005","is_oa":true,"landing_page_url":"https://doi.org/10.3390/software4010005","pdf_url":"https://www.mdpi.com/2674-113X/4/1/5/pdf?version=1741599844","source":{"id":"https://openalex.org/S4210183365","display_name":"Software","issn_l":"2674-113X","issn":["2674-113X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4408290845.pdf"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W1971650562","https://openalex.org/W1974571153","https://openalex.org/W1979345446","https://openalex.org/W2041176136","https://openalex.org/W2122205205","https://openalex.org/W2158373592","https://openalex.org/W2287823335","https://openalex.org/W2288977256","https://openalex.org/W2682664750","https://openalex.org/W2882984136","https://openalex.org/W3082051432","https://openalex.org/W3093870681","https://openalex.org/W3124323960","https://openalex.org/W3135005174","https://openalex.org/W4362508616","https://openalex.org/W4367860052","https://openalex.org/W4384345649","https://openalex.org/W4392157268","https://openalex.org/W4398239248","https://openalex.org/W4399668074","https://openalex.org/W4400582690","https://openalex.org/W4401996408","https://openalex.org/W4402671827","https://openalex.org/W4403447349","https://openalex.org/W4403536268","https://openalex.org/W6651102376","https://openalex.org/W6683841033","https://openalex.org/W6695910771","https://openalex.org/W6852275782"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Software":[0],"testing":[1],"ensures":[2],"the":[3,18,45,58,75,176],"quality":[4],"and":[5,67,81,103,124,178,189],"reliability":[6],"of":[7,20,38,47,60,115,180],"software":[8],"products,":[9],"but":[10,167],"manual":[11],"test":[12,31,40,63,95,183],"case":[13,64],"creation":[14,32],"is":[15,26,42],"labor-intensive.":[16],"With":[17],"rise":[19],"Large":[21],"Language":[22],"Models":[23],"(LLMs),":[24],"there":[25],"growing":[27],"interest":[28],"in":[29,164,182],"unit":[30],"with":[33,170],"LLMs.":[34],"However,":[35],"effective":[36],"assessment":[37,59],"LLM-generated":[39],"cases":[41],"limited":[43],"by":[44],"lack":[46],"standardized":[48],"benchmarks":[49],"that":[50,149],"comprehensively":[51],"cover":[52],"diverse":[53],"programming":[54],"scenarios.":[55],"To":[56],"address":[57],"an":[61],"LLM\u2019s":[62],"generation":[65,96],"ability":[66],"lacking":[68],"a":[69,108,113,186],"dataset":[70],"for":[71,92,192],"evaluation,":[72],"we":[73,137],"propose":[74],"Generated":[76],"Benchmark":[77],"from":[78,118],"Control-Flow":[79],"Structure":[80],"Variable":[82],"Usage":[83],"Composition":[84],"(GBCV)":[85],"approach,":[86],"which":[87],"systematically":[88],"generates":[89],"programs":[90,116],"used":[91],"evaluating":[93],"LLMs\u2019":[94],"capabilities.":[97],"By":[98],"leveraging":[99],"basic":[100],"control-flow":[101],"structures":[102],"variable":[104],"usage,":[105],"GBCV":[106,139],"provides":[107,185],"flexible":[109],"framework":[110],"to":[111,120,130,140],"create":[112],"spectrum":[114],"ranging":[117],"simple":[119,165],"complex.":[121],"Because":[122],"GPT-4o":[123,150],"GPT-3.5-Turbo":[125],"are":[126],"publicly":[127],"accessible":[128],"models,":[129],"present":[131],"real-world":[132],"regular":[133],"users\u2019":[134],"use":[135,138],"cases,":[136],"assess":[141],"LLM":[142],"performance":[143],"on":[144,153],"them.":[145],"Our":[146],"findings":[147],"indicate":[148],"performs":[151],"better":[152],"composite":[154],"program":[155],"structures,":[156],"while":[157],"all":[158],"models":[159],"effectively":[160],"detect":[161],"boundary":[162],"values":[163],"conditions":[166],"face":[168],"challenges":[169],"arithmetic":[171],"computations.":[172],"This":[173],"study":[174],"highlights":[175],"strengths":[177],"limitations":[179],"LLMs":[181],"generation,":[184],"benchmark":[187],"framework,":[188],"suggests":[190],"directions":[191],"future":[193],"improvement.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4}],"updated_date":"2026-02-27T16:54:17.756197","created_date":"2025-10-10T00:00:00"}