{"id":"https://openalex.org/W4391974543","doi":"https://doi.org/10.1109/tse.2024.3368208","title":"Software Testing With Large Language Models: Survey, Landscape, and Vision","display_name":"Software Testing With Large Language Models: Survey, Landscape, and Vision","publication_year":2024,"publication_date":"2024-02-20","ids":{"openalex":"https://openalex.org/W4391974543","doi":"https://doi.org/10.1109/tse.2024.3368208"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3368208","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3368208","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108814327","display_name":"Junjie Wang","orcid":"https://orcid.org/0000-0002-9941-6713"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Junjie Wang","raw_affiliation_strings":["State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9941-6713","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056375705","display_name":"Yuchao Huang","orcid":"https://orcid.org/0000-0003-1026-9516"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuchao Huang","raw_affiliation_strings":["State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1026-9516","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075639297","display_name":"Chunyang Chen","orcid":"https://orcid.org/0000-0003-2011-9618"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Chunyang Chen","raw_affiliation_strings":["Technical University of Munich, Munich, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2011-9618","affiliations":[{"raw_affiliation_string":"Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100462353","display_name":"Zhe Liu","orcid":"https://orcid.org/0000-0001-9709-8275"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhe Liu","raw_affiliation_strings":["State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9709-8275","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100326213","display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-0531-5717"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Song Wang","raw_affiliation_strings":["York University, Toronto, ON, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0531-5717","affiliations":[{"raw_affiliation_string":"York University, Toronto, ON, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100434820","display_name":"Qing Wang","orcid":"https://orcid.org/0000-0002-2618-5694"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qing Wang","raw_affiliation_strings":["State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2618-5694","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Intelligent Game, Institute of Software Chinese Academy of Sciences, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5108814327"],"corresponding_institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I4210165038"],"apc_list":null,"apc_paid":null,"fwci":169.0681,"has_fulltext":false,"cited_by_count":373,"citation_normalized_percentile":{"value":1.0,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"50","issue":"4","first_page":"911","last_page":"936"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6445666551589966},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4440211057662964},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.43670931458473206}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6445666551589966},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4440211057662964},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.43670931458473206},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2024.3368208","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3368208","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1581568102","display_name":null,"funder_award_id":"62072442","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5717671410","display_name":null,"funder_award_id":"62272445","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6800602771","display_name":null,"funder_award_id":"62232016","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":220,"referenced_works":["https://openalex.org/W5399261","https://openalex.org/W967544008","https://openalex.org/W1524053243","https://openalex.org/W1975675278","https://openalex.org/W2027422213","https://openalex.org/W2041713059","https://openalex.org/W2107709519","https://openalex.org/W2109596254","https://openalex.org/W2282179196","https://openalex.org/W2344973853","https://openalex.org/W2506155917","https://openalex.org/W2518136680","https://openalex.org/W2621123093","https://openalex.org/W2740742367","https://openalex.org/W2795030435","https://openalex.org/W2867448323","https://openalex.org/W2899369852","https://openalex.org/W3007157104","https://openalex.org/W3015693366","https://openalex.org/W3043122026","https://openalex.org/W3084812981","https://openalex.org/W3086938529","https://openalex.org/W3093605317","https://openalex.org/W3104663419","https://openalex.org/W3154106427","https://openalex.org/W3161997752","https://openalex.org/W3162089186","https://openalex.org/W3195818176","https://openalex.org/W3201342367","https://openalex.org/W4206238894","https://openalex.org/W4220699888","https://openalex.org/W4221157342","https://openalex.org/W4224075202","https://openalex.org/W4281479826","https://openalex.org/W4283371798","https://openalex.org/W4284686707","https://openalex.org/W4285141508","https://openalex.org/W4285204876","https://openalex.org/W4285490440","https://openalex.org/W4286750487","https://openalex.org/W4287668913","https://openalex.org/W4291238279","https://openalex.org/W4298186479","https://openalex.org/W4303648699","https://openalex.org/W4308641648","https://openalex.org/W4310923406","https://openalex.org/W4312565886","https://openalex.org/W4312669928","https://openalex.org/W4312962198","https://openalex.org/W4312970618","https://openalex.org/W4313059319","https://openalex.org/W4315589210","https://openalex.org/W4319240918","https://openalex.org/W4362508616","https://openalex.org/W4362515116","https://openalex.org/W4362598291","https://openalex.org/W4362679230","https://openalex.org/W4365143346","https://openalex.org/W4365205411","https://openalex.org/W4366327299","https://openalex.org/W4366851162","https://openalex.org/W4367046771","https://openalex.org/W4367060696","https://openalex.org/W4375840656","https://openalex.org/W4375959406","https://openalex.org/W4376122390","https://openalex.org/W4376167213","https://openalex.org/W4376311899","https://openalex.org/W4376606826","https://openalex.org/W4376988928","https://openalex.org/W4378473878","https://openalex.org/W4378591002","https://openalex.org/W4378647841","https://openalex.org/W4378942602","https://openalex.org/W4380558484","https://openalex.org/W4382406517","https://openalex.org/W4382766512","https://openalex.org/W4383175415","https://openalex.org/W4383988989","https://openalex.org/W4384026553","https://openalex.org/W4384159069","https://openalex.org/W4384163464","https://openalex.org/W4384302749","https://openalex.org/W4384302785","https://openalex.org/W4384304728","https://openalex.org/W4384304865","https://openalex.org/W4384345649","https://openalex.org/W4384345664","https://openalex.org/W4384345667","https://openalex.org/W4384345708","https://openalex.org/W4384345745","https://openalex.org/W4384345748","https://openalex.org/W4385261653","https://openalex.org/W4385302156","https://openalex.org/W4385302218","https://openalex.org/W4385365454","https://openalex.org/W4385500670","https://openalex.org/W4385572142","https://openalex.org/W4385573988","https://openalex.org/W4385774599","https://openalex.org/W4385774908","https://openalex.org/W4385848840","https://openalex.org/W4385965483","https://openalex.org/W4386081014","https://openalex.org/W4386148033","https://openalex.org/W4386231786","https://openalex.org/W4386273180","https://openalex.org/W4386302809","https://openalex.org/W4386436496","https://openalex.org/W4387031659","https://openalex.org/W4387323527","https://openalex.org/W4387355106","https://openalex.org/W4387389705","https://openalex.org/W4387560977","https://openalex.org/W4387711873","https://openalex.org/W4387793519","https://openalex.org/W4387947606","https://openalex.org/W4388093248","https://openalex.org/W4388095126","https://openalex.org/W4388483128","https://openalex.org/W4388483194","https://openalex.org/W4388483238","https://openalex.org/W4388483492","https://openalex.org/W4388483640","https://openalex.org/W4388685483","https://openalex.org/W4388717680","https://openalex.org/W4389104713","https://openalex.org/W4389158474","https://openalex.org/W4389162178","https://openalex.org/W4389285870","https://openalex.org/W4389520282","https://openalex.org/W4390188013","https://openalex.org/W4390692489","https://openalex.org/W4391558545","https://openalex.org/W4391579639","https://openalex.org/W4391579699","https://openalex.org/W4392414327","https://openalex.org/W4393305539","https://openalex.org/W4394015350","https://openalex.org/W4394745107","https://openalex.org/W4394769228","https://openalex.org/W4394769323","https://openalex.org/W4396833734","https://openalex.org/W4398239248","https://openalex.org/W4399361983","https://openalex.org/W4399557965","https://openalex.org/W4401996408","https://openalex.org/W4402042086","https://openalex.org/W4402410897","https://openalex.org/W4402665833","https://openalex.org/W6625168331","https://openalex.org/W6783227185","https://openalex.org/W6783930131","https://openalex.org/W6786111318","https://openalex.org/W6795855765","https://openalex.org/W6809646742","https://openalex.org/W6809811189","https://openalex.org/W6838865847","https://openalex.org/W6840793632","https://openalex.org/W6841581684","https://openalex.org/W6841881191","https://openalex.org/W6843977082","https://openalex.org/W6845906316","https://openalex.org/W6847505539","https://openalex.org/W6849180489","https://openalex.org/W6849340928","https://openalex.org/W6851089358","https://openalex.org/W6851228291","https://openalex.org/W6851560035","https://openalex.org/W6851748684","https://openalex.org/W6851775633","https://openalex.org/W6851797817","https://openalex.org/W6851935182","https://openalex.org/W6851944205","https://openalex.org/W6852119581","https://openalex.org/W6852126889","https://openalex.org/W6852256804","https://openalex.org/W6852294948","https://openalex.org/W6852306619","https://openalex.org/W6852417946","https://openalex.org/W6852439742","https://openalex.org/W6852512558","https://openalex.org/W6852775702","https://openalex.org/W6852896372","https://openalex.org/W6853582562","https://openalex.org/W6853867249","https://openalex.org/W6853893586","https://openalex.org/W6853945670","https://openalex.org/W6854027915","https://openalex.org/W6854538240","https://openalex.org/W6854746895","https://openalex.org/W6855534052","https://openalex.org/W6855687073","https://openalex.org/W6855754046","https://openalex.org/W6855807987","https://openalex.org/W6855856048","https://openalex.org/W6855930829","https://openalex.org/W6856051534","https://openalex.org/W6856074014","https://openalex.org/W6856367867","https://openalex.org/W6856504490","https://openalex.org/W6856556076","https://openalex.org/W6856646429","https://openalex.org/W6856707112","https://openalex.org/W6856763231","https://openalex.org/W6856923314","https://openalex.org/W6857028926","https://openalex.org/W6857241383","https://openalex.org/W6857245479","https://openalex.org/W6857292862","https://openalex.org/W6857345414","https://openalex.org/W6857445007","https://openalex.org/W6857485932","https://openalex.org/W6857719831","https://openalex.org/W6857854514","https://openalex.org/W6858033984","https://openalex.org/W6858801840","https://openalex.org/W6859827566","https://openalex.org/W6859866797","https://openalex.org/W7036171765"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Pre-trained":[0],"large":[1],"language":[2,14],"models":[3],"(LLMs)":[4],"have":[5,113],"recently":[6],"emerged":[7],"as":[8,45,88,171,173,196],"a":[9,31,40,46,96,130,197],"breakthrough":[10],"technology":[11],"in":[12,104,189,202,213,222],"natural":[13],"processing":[15],"and":[16,26,52,60,124,149,186,210],"artificial":[17],"intelligence,":[18],"with":[19,177],"the":[20,50,58,68,89,100,121,134,153,159,163,174,183,218],"ability":[21],"to":[22,66],"handle":[23],"large-scale":[24],"datasets":[25],"exhibit":[27],"remarkable":[28],"performance":[29],"across":[30],"wide":[32],"range":[33],"of":[34,54,62,91,99,102,133,165,217,220],"tasks.":[35],"Meanwhile,":[36],"software":[37,55,63,73,105,117,122,135,223],"testing":[38,74,123,136],"is":[39],"crucial":[41],"undertaking":[42],"that":[43,112,168],"serves":[44],"cornerstone":[47],"for":[48,70,84,116,138,199,208],"ensuring":[49],"quality":[51],"reliability":[53],"products.":[56],"As":[57],"scope":[59],"complexity":[61],"systems":[64],"continue":[65],"grow,":[67],"need":[69],"more":[71],"effective":[72],"techniques":[75,176],"becomes":[76],"increasingly":[77],"urgent,":[78],"making":[79],"it":[80],"an":[81],"area":[82],"ripe":[83],"innovative":[85],"approaches":[86],"such":[87],"use":[90,219],"LLMs.":[92,179],"This":[93,192],"paper":[94,128],"provides":[95],"comprehensive":[97],"review":[98],"utilization":[101],"LLMs":[103,115,125,140,221],"testing.":[106,224],"It":[107,156,180],"analyzes":[108,158],"102":[109],"relevant":[110],"studies":[111],"used":[114,161],"testing,":[118],"from":[119],"both":[120],"perspectives.":[126],"The":[127],"presents":[129],"detailed":[131],"discussion":[132],"tasks":[137],"which":[139,145],"are":[141,152,169],"commonly":[142,160],"used,":[143],"among":[144],"test":[146],"case":[147],"preparation":[148],"program":[150],"repair":[151],"most":[154],"representative.":[155],"also":[157,181],"LLMs,":[162],"types":[164],"prompt":[166],"engineering":[167],"employed,":[170],"well":[172],"accompanied":[175],"these":[178],"summarizes":[182],"key":[184],"challenges":[185],"potential":[187,206],"opportunities":[188],"this":[190,203],"direction.":[191],"work":[193],"can":[194],"serve":[195],"roadmap":[198],"future":[200],"research":[201],"area,":[204],"highlighting":[205],"avenues":[207],"exploration,":[209],"identifying":[211],"gaps":[212],"our":[214],"current":[215],"understanding":[216]},"counts_by_year":[{"year":2026,"cited_by_count":47},{"year":2025,"cited_by_count":255},{"year":2024,"cited_by_count":71}],"updated_date":"2026-05-13T08:25:38.343686","created_date":"2025-10-10T00:00:00"}