{"id":"https://openalex.org/W7117156918","doi":"https://doi.org/10.1145/3756681.3756964","title":"Quality Assessment of Python Tests Generated by Large Language Models","display_name":"Quality Assessment of Python Tests Generated by Large Language Models","publication_year":2025,"publication_date":"2025-06-17","ids":{"openalex":"https://openalex.org/W7117156918","doi":"https://doi.org/10.1145/3756681.3756964"},"language":null,"primary_location":{"id":"doi:10.1145/3756681.3756964","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3756681.3756964","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3756681.3756964","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5121125979","display_name":"Victor Alves","orcid":null},"institutions":[{"id":"https://openalex.org/I243754102","display_name":"Universidade Federal do Cear\u00e1","ror":"https://ror.org/03srtnf24","country_code":"BR","type":"education","lineage":["https://openalex.org/I243754102"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Victor Alves","raw_affiliation_strings":["Federal University of Cear\u00e1 (UFC), Quixad\u00e1, Cear\u00e1, Brazil"],"raw_orcid":"https://orcid.org/0009-0006-6583-5350","affiliations":[{"raw_affiliation_string":"Federal University of Cear\u00e1 (UFC), Quixad\u00e1, Cear\u00e1, Brazil","institution_ids":["https://openalex.org/I243754102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121229110","display_name":"Carla Bezerra","orcid":null},"institutions":[{"id":"https://openalex.org/I243754102","display_name":"Universidade Federal do Cear\u00e1","ror":"https://ror.org/03srtnf24","country_code":"BR","type":"education","lineage":["https://openalex.org/I243754102"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Carla Bezerra","raw_affiliation_strings":["Federal University of Cear\u00e1 (UFC), Quixad\u00e1, Cear\u00e1, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-5879-5067","affiliations":[{"raw_affiliation_string":"Federal University of Cear\u00e1 (UFC), Quixad\u00e1, Cear\u00e1, Brazil","institution_ids":["https://openalex.org/I243754102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121216447","display_name":"Ivan Machado","orcid":null},"institutions":[{"id":"https://openalex.org/I126158947","display_name":"Universidade Federal da Bahia","ror":"https://ror.org/03k3p7647","country_code":"BR","type":"education","lineage":["https://openalex.org/I126158947"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Ivan Machado","raw_affiliation_strings":["Fedeal University of Bahia (UFBA), Salvador, Brazil"],"raw_orcid":"https://orcid.org/0000-0001-9027-2293","affiliations":[{"raw_affiliation_string":"Fedeal University of Bahia (UFBA), Salvador, Brazil","institution_ids":["https://openalex.org/I126158947"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121198470","display_name":"Larissa Rocha","orcid":null},"institutions":[{"id":"https://openalex.org/I1316675753","display_name":"Universidade do Estado da Bahia","ror":"https://ror.org/015n1m812","country_code":"BR","type":"education","lineage":["https://openalex.org/I1316675753"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Larissa Rocha","raw_affiliation_strings":["University of the State of Bahia (UNEB), Alagoinhas, Bahia, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-8069-5249","affiliations":[{"raw_affiliation_string":"University of the State of Bahia (UNEB), Alagoinhas, Bahia, Brazil","institution_ids":["https://openalex.org/I1316675753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030148545","display_name":"T\u00e1ssio Virg\u00ednio","orcid":"https://orcid.org/0000-0001-6259-4957"},"institutions":[{"id":"https://openalex.org/I126158947","display_name":"Universidade Federal da Bahia","ror":"https://ror.org/03k3p7647","country_code":"BR","type":"education","lineage":["https://openalex.org/I126158947"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Tassio Virg\u00ednio","raw_affiliation_strings":["Federal University of Bahia (UFBA), Salvador, Bahia, Brazil"],"raw_orcid":"https://orcid.org/0000-0001-6259-4957","affiliations":[{"raw_affiliation_string":"Federal University of Bahia (UFBA), Salvador, Bahia, Brazil","institution_ids":["https://openalex.org/I126158947"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079998712","display_name":"P\u00fablio Silva","orcid":"https://orcid.org/0000-0002-1655-4476"},"institutions":[{"id":"https://openalex.org/I243754102","display_name":"Universidade Federal do Cear\u00e1","ror":"https://ror.org/03srtnf24","country_code":"BR","type":"education","lineage":["https://openalex.org/I243754102"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Publio Silva","raw_affiliation_strings":["Federal University of Cear\u00e1 (UFC), Quixad\u00e1, Cear\u00e1, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-1655-4476","affiliations":[{"raw_affiliation_string":"Federal University of Cear\u00e1 (UFC), Quixad\u00e1, Cear\u00e1, Brazil","institution_ids":["https://openalex.org/I243754102"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"394","last_page":"405"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7163000106811523,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7163000106811523,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.11509999632835388,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.014600000344216824,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.6165000200271606},{"id":"https://openalex.org/keywords/assertion","display_name":"Assertion","score":0.5256999731063843},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.47850000858306885},{"id":"https://openalex.org/keywords/test-script","display_name":"Test script","score":0.4702000021934509},{"id":"https://openalex.org/keywords/cohesion","display_name":"Cohesion (chemistry)","score":0.46779999136924744},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.4560000002384186},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.41670000553131104},{"id":"https://openalex.org/keywords/quality-assessment","display_name":"Quality assessment","score":0.3758000135421753}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6506999731063843},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.6165000200271606},{"id":"https://openalex.org/C40422974","wikidata":"https://www.wikidata.org/wiki/Q741248","display_name":"Assertion","level":2,"score":0.5256999731063843},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.47850000858306885},{"id":"https://openalex.org/C109086967","wikidata":"https://www.wikidata.org/wiki/Q2509100","display_name":"Test script","level":4,"score":0.4702000021934509},{"id":"https://openalex.org/C104054115","wikidata":"https://www.wikidata.org/wiki/Q216828","display_name":"Cohesion (chemistry)","level":2,"score":0.46779999136924744},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.4560000002384186},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.41670000553131104},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.40529999136924744},{"id":"https://openalex.org/C3020001037","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assessment","level":3,"score":0.3758000135421753},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.35089999437332153},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3434000015258789},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3370000123977661},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3276999890804291},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.3037000000476837},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C7435765","wikidata":"https://www.wikidata.org/wiki/Q7705776","display_name":"Test Management Approach","level":5,"score":0.29789999127388},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.29660001397132874},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2930999994277954},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.27889999747276306},{"id":"https://openalex.org/C535291247","wikidata":"https://www.wikidata.org/wiki/Q177567","display_name":"Amazon rainforest","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.27230000495910645},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.27160000801086426},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.26080000400543213},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.2533999979496002}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3756681.3756964","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3756681.3756964","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3756681.3756964","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3756681.3756964","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G371314912","display_name":null,"funder_award_id":"403361/2023-0","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G8281696261","display_name":null,"funder_award_id":"315840/2023-4","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"}],"funders":[{"id":"https://openalex.org/F4320322025","display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","ror":"https://ror.org/03swz6y49"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W2002753905","https://openalex.org/W2114869486","https://openalex.org/W2131507481","https://openalex.org/W2508117657","https://openalex.org/W2611190712","https://openalex.org/W2807453842","https://openalex.org/W2900471508","https://openalex.org/W3091625722","https://openalex.org/W3113869096","https://openalex.org/W3155252986","https://openalex.org/W4205239483","https://openalex.org/W4285490448","https://openalex.org/W4296535233","https://openalex.org/W4308627320","https://openalex.org/W4312377019","https://openalex.org/W4366851162","https://openalex.org/W4382652775","https://openalex.org/W4387736592","https://openalex.org/W4389104713","https://openalex.org/W4389889093","https://openalex.org/W4390188013","https://openalex.org/W4390638022","https://openalex.org/W4396788588","https://openalex.org/W4399491517","https://openalex.org/W4399667811","https://openalex.org/W4399668074","https://openalex.org/W4400484392","https://openalex.org/W4400484796","https://openalex.org/W4400582872","https://openalex.org/W4400681667","https://openalex.org/W4400878080","https://openalex.org/W4403536220","https://openalex.org/W4403536388","https://openalex.org/W4403584988","https://openalex.org/W4403607016"],"related_works":[],"abstract_inverted_index":{"The":[0],"manual":[1],"generation":[2,257,270],"of":[3,16,45,64,83,126,135,137,167,255],"test":[4,36,47,65,86,104,116,132,150,168,205,243,256],"scripts":[5],"is":[6],"a":[7,89,164,230],"time-intensive,":[8],"costly,":[9],"and":[10,56,75,85,185,200,242,260,272],"error-prone":[11],"process,":[12],"indicating":[13],"the":[14,43,61,81,108,121,131,141,171,176,193,213,223,253,262],"value":[15],"automated":[17],"solutions.":[18],"Large":[19],"Language":[20],"Models":[21],"(LLMs)":[22],"have":[23],"shown":[24],"great":[25],"promise":[26],"in":[27,179,183,187,198,202,212,222],"this":[28],"domain,":[29],"leveraging":[30],"their":[31],"extensive":[32],"knowledge":[33],"to":[34,95,267],"produce":[35],"code":[37,48],"more":[38],"efficiently.":[39],"This":[40],"study":[41],"investigates":[42],"quality":[44,254],"Python":[46],"generated":[49,67,106],"by":[50,107,258],"three":[51],"LLMs:":[52],"GPT-4o,":[53],"Amazon":[54,190,207],"Q,":[55],"LLama":[57,218],"3.3.":[58],"We":[59],"evaluate":[60],"structural":[62],"reliability":[63],"suites":[66,105],"under":[68],"two":[69],"distinct":[70],"prompt":[71,274],"contexts:":[72],"Text2Code":[73],"(T2C)":[74],"Code2Code":[76],"(C2C).":[77],"Our":[78,99],"analysis":[79],"includes":[80],"identification":[82],"errors":[84,119,162,178],"smells,":[87,206],"with":[88,154,160],"focus":[90],"on":[91],"correlating":[92],"these":[93],"issues":[94],"inadequate":[96],"design":[97],"patterns.":[98],"findings":[100,248],"reveal":[101],"that":[102],"most":[103,122,142],"LLMs":[109,259],"contained":[110],"at":[111],"least":[112],"one":[113],"error":[114,195],"or":[115,239],"smell.":[117],"Assertion":[118],"were":[120],"common,":[123],"comprising":[124],"64%":[125],"all":[127],"identified":[128],"errors,":[129,235],"while":[130,217],"smell":[133],"Lack":[134],"Cohesion":[136],"Test":[138],"Cases":[139],"was":[140],"frequently":[143],"detected":[144],"(41%).":[145],"Prompt":[146],"context":[147,215,225],"significantly":[148],"influenced":[149],"quality;":[151],"textual":[152],"prompts":[153],"detailed":[155],"instructions":[156],"often":[157],"yielded":[158],"tests":[159],"fewer":[161,210],"but":[163],"higher":[165],"incidence":[166],"smells.":[169,246],"Among":[170],"evaluated":[172],"LLMs,":[173],"GPT-4o":[174],"produced":[175],"fewest":[177],"both":[180],"contexts":[181],"(10%":[182],"C2C":[184,199,214],"6%":[186],"T2C),":[188],"whereas":[189],"Q":[191,208],"had":[192,209],"highest":[194],"rates":[196],"(19%":[197],"28%":[201],"T2C).":[203],"For":[204],"detections":[211],"(9%),":[216],"3.3":[219],"performed":[220],"best":[221],"T2C":[224],"(10%).":[226],"Additionally,":[227],"we":[228],"observed":[229],"strong":[231],"relationship":[232],"between":[233],"specific":[234],"such":[236],"as":[237],"assertion":[238],"indentation":[240],"issues,":[241],"case":[244],"cohesion":[245],"These":[247],"demonstrate":[249],"opportunities":[250],"for":[251,264],"improving":[252],"highlight":[261],"need":[263],"future":[265],"research":[266],"explore":[268],"optimized":[269],"scenarios":[271],"better":[273],"engineering":[275],"strategies.":[276]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-12-24T00:00:00"}