{"id":"https://openalex.org/W4411449925","doi":"https://doi.org/10.1145/3715752","title":"Automated Soap Opera Testing Directed by LLMs and Scenario Knowledge: Feasibility, Challenges, and Road Ahead","display_name":"Automated Soap Opera Testing Directed by LLMs and Scenario Knowledge: Feasibility, Challenges, and Road Ahead","publication_year":2025,"publication_date":"2025-06-19","ids":{"openalex":"https://openalex.org/W4411449925","doi":"https://doi.org/10.1145/3715752"},"language":"en","primary_location":{"id":"doi:10.1145/3715752","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3715752","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086023162","display_name":"Yanqi Su","orcid":"https://orcid.org/0000-0002-2410-3229"},"institutions":[{"id":"https://openalex.org/I118347636","display_name":"Australian National University","ror":"https://ror.org/019wvm592","country_code":"AU","type":"education","lineage":["https://openalex.org/I118347636"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Yanqi Su","raw_affiliation_strings":["Australian National University, Canberra, Australia"],"affiliations":[{"raw_affiliation_string":"Australian National University, Canberra, Australia","institution_ids":["https://openalex.org/I118347636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028641941","display_name":"Zhenchang Xing","orcid":"https://orcid.org/0000-0001-7663-1421"},"institutions":[{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"funder","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Zhenchang Xing","raw_affiliation_strings":["CSIRO's Data61, Canberra, Australia"],"affiliations":[{"raw_affiliation_string":"CSIRO's Data61, Canberra, Australia","institution_ids":["https://openalex.org/I42894916","https://openalex.org/I1292875679"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100329466","display_name":"Chong Wang","orcid":"https://orcid.org/0000-0003-1424-6290"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Chong Wang","raw_affiliation_strings":["Nanyang Technological University, Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075639297","display_name":"Chunyang Chen","orcid":"https://orcid.org/0000-0003-2011-9618"},"institutions":[{"id":"https://openalex.org/I41121874","display_name":"Heilbronn University","ror":"https://ror.org/04g5gcg95","country_code":"DE","type":"education","lineage":["https://openalex.org/I41121874"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Chunyang Chen","raw_affiliation_strings":["TU Munich, Heilbronn, Germany"],"affiliations":[{"raw_affiliation_string":"TU Munich, Heilbronn, Germany","institution_ids":["https://openalex.org/I41121874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006841485","display_name":"Xiwei Xu","orcid":"https://orcid.org/0000-0002-2273-1862"},"institutions":[{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"funder","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Sherry (Xiwei) Xu","raw_affiliation_strings":["CSIRO's Data61, Sydney, Australia"],"affiliations":[{"raw_affiliation_string":"CSIRO's Data61, Sydney, Australia","institution_ids":["https://openalex.org/I42894916","https://openalex.org/I1292875679"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100652461","display_name":"Qinghua Lu","orcid":"https://orcid.org/0000-0002-9466-1672"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"funder","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Qinghua Lu","raw_affiliation_strings":["CSIRO, Sydney, Australia"],"affiliations":[{"raw_affiliation_string":"CSIRO, Sydney, Australia","institution_ids":["https://openalex.org/I1292875679"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064683660","display_name":"Liming Zhu","orcid":"https://orcid.org/0000-0001-5839-3765"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"funder","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]},{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Liming Zhu","raw_affiliation_strings":["CSIRO's Data61, Sydney, Australia","UNSW, Sydney, Australia"],"affiliations":[{"raw_affiliation_string":"CSIRO's Data61, Sydney, Australia","institution_ids":["https://openalex.org/I42894916","https://openalex.org/I1292875679"]},{"raw_affiliation_string":"UNSW, Sydney, Australia","institution_ids":["https://openalex.org/I31746571"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5086023162"],"corresponding_institution_ids":["https://openalex.org/I118347636"],"apc_list":null,"apc_paid":null,"fwci":2.4061,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.87907088,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"2","issue":"FSE","first_page":"757","last_page":"778"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/opera","display_name":"Opera","score":0.6933341026306152},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6304831504821777},{"id":"https://openalex.org/keywords/soap","display_name":"SOAP","score":0.5874994397163391},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.5184821486473083},{"id":"https://openalex.org/keywords/planner","display_name":"Planner","score":0.5178759098052979},{"id":"https://openalex.org/keywords/formative-assessment","display_name":"Formative assessment","score":0.49779319763183594},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.45311155915260315},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.2941837012767792},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.27082642912864685},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2646386921405792}],"concepts":[{"id":"https://openalex.org/C530479602","wikidata":"https://www.wikidata.org/wiki/Q1344","display_name":"Opera","level":2,"score":0.6933341026306152},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6304831504821777},{"id":"https://openalex.org/C17881449","wikidata":"https://www.wikidata.org/wiki/Q189620","display_name":"SOAP","level":2,"score":0.5874994397163391},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.5184821486473083},{"id":"https://openalex.org/C2776999362","wikidata":"https://www.wikidata.org/wiki/Q2349274","display_name":"Planner","level":2,"score":0.5178759098052979},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.49779319763183594},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.45311155915260315},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2941837012767792},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.27082642912864685},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2646386921405792},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3715752","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3715752","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/201467","is_oa":false,"landing_page_url":"https://hdl.handle.net/10356/201467","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W613156","https://openalex.org/W579397108","https://openalex.org/W609613424","https://openalex.org/W1481784587","https://openalex.org/W1985308454","https://openalex.org/W1988134631","https://openalex.org/W2018567346","https://openalex.org/W2087287766","https://openalex.org/W2108555190","https://openalex.org/W2186046355","https://openalex.org/W2481844499","https://openalex.org/W2789020245","https://openalex.org/W2854345383","https://openalex.org/W2909014315","https://openalex.org/W3027879771","https://openalex.org/W3027993363","https://openalex.org/W3090875140","https://openalex.org/W3090921344","https://openalex.org/W3100403944","https://openalex.org/W3101955389","https://openalex.org/W3114863449","https://openalex.org/W3160758932","https://openalex.org/W4220979144","https://openalex.org/W4239249613","https://openalex.org/W4284676027","https://openalex.org/W4312763760","https://openalex.org/W4313547594","https://openalex.org/W4366549767","https://openalex.org/W4384155409","https://openalex.org/W4384302749","https://openalex.org/W4384337827","https://openalex.org/W4384345667","https://openalex.org/W4385152093","https://openalex.org/W4394769228","https://openalex.org/W4394769383","https://openalex.org/W4396833734","https://openalex.org/W4399657679"],"related_works":["https://openalex.org/W4392721137","https://openalex.org/W609482051","https://openalex.org/W3199325201","https://openalex.org/W2792035366","https://openalex.org/W1580704634","https://openalex.org/W1504937917","https://openalex.org/W3088407659","https://openalex.org/W2045536557","https://openalex.org/W1965119092","https://openalex.org/W4244027346"],"abstract_inverted_index":{"Exploratory":[0],"testing":[1,28,73,183],"(ET)":[2],"harnesses":[3],"tester's":[4],"knowledge,":[5],"creativity,":[6],"and":[7,49,74,88,108,115,143,172,177,194],"experience":[8],"to":[9,94,112,136,192],"create":[10],"varying":[11],"tests":[12,114],"that":[13,110],"uncover":[14],"unexpected":[15],"bugs":[16],"from":[17],"the":[18,32,46,78,122,149,156,168,178,196],"end-user's":[19],"perspective.":[20],"Although":[21],"ET":[22,55],"has":[23,37],"proven":[24],"effective":[25,69],"in":[26,76],"system-level":[27],"of":[29,52,102,124,158,170,180],"interactive":[30],"systems,":[31],"need":[33],"for":[34,68,155],"manual":[35,70,137],"execution":[36],"hindered":[38],"large-scale":[39],"adoption.":[40],"In":[41],"this":[42],"work,":[43],"we":[44,151],"explore":[45],"feasibility,":[47],"challenges":[48,75],"road":[50,153],"ahead":[51,154],"automated":[53,125,159],"scenario-based":[54],"(a.k.a":[56],"soap":[57,71,96,126,160,181],"opera":[58,72,97,127,161,182],"testing).":[59],"We":[60,80],"conduct":[61],"a":[62,83,89,132],"formative":[63],"study,":[64],"identifying":[65],"key":[66,166],"insights":[67,190],"automating":[77],"process.":[79],"then":[81],"develop":[82],"multi-agent":[84],"system":[85,100],"leveraging":[86],"LLMs":[87],"Scenario":[90],"Knowledge":[91],"Graph":[92],"(SKG)":[93],"automate":[95],"testing.":[98],"The":[99],"consists":[101],"three":[103,165],"multi-modal":[104],"agents,":[105],"Planner,":[106],"Player,":[107],"Detector":[109],"collaborate":[111],"execute":[113],"identify":[116],"potential":[117,123],"bugs.":[118,146],"Experimental":[119],"results":[120],"demonstrate":[121],"testing,":[128,162],"but":[129],"there":[130],"remains":[131],"significant":[133],"gap":[134],"compared":[135],"execution,":[138],"especially":[139],"under-explored":[140],"scenario":[141],"boundaries":[142],"incorrectly":[144],"identified":[145],"Based":[147],"on":[148,164],"observation,":[150],"envision":[152],"future":[157,197],"focusing":[163],"aspects:":[167],"synergy":[169],"neural":[171],"symbolic":[173],"approaches,":[174],"human-AI":[175],"co-learning,":[176],"integration":[179],"with":[184],"broader":[185],"software":[186],"engineering":[187],"practices.":[188],"These":[189],"aim":[191],"guide":[193],"inspire":[195],"research.":[198]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}