{"id":"https://openalex.org/W4415034613","doi":"https://doi.org/10.1109/scam67354.2025.00015","title":"Testing the Untestable? An Empirical Study on the Testing Process of LLM-Powered Software Systems","display_name":"Testing the Untestable? An Empirical Study on the Testing Process of LLM-Powered Software Systems","publication_year":2025,"publication_date":"2025-09-08","ids":{"openalex":"https://openalex.org/W4415034613","doi":"https://doi.org/10.1109/scam67354.2025.00015"},"language":"en","primary_location":{"id":"doi:10.1109/scam67354.2025.00015","is_oa":false,"landing_page_url":"https://doi.org/10.1109/scam67354.2025.00015","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Source Code Analysis & Manipulation (SCAM)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046194884","display_name":"Cleyton Magalh\u00e3es","orcid":"https://orcid.org/0009-0005-3051-7232"},"institutions":[{"id":"https://openalex.org/I62921916","display_name":"Universidade Federal Rural de Pernambuco","ror":"https://ror.org/02ksmb993","country_code":"BR","type":"education","lineage":["https://openalex.org/I62921916"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Cleyton Magalhaes","raw_affiliation_strings":["UFRPE,Recife,PE,Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFRPE,Recife,PE,Brazil","institution_ids":["https://openalex.org/I62921916"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021360591","display_name":"\u00cdtalo Santos","orcid":"https://orcid.org/0000-0002-7545-6104"},"institutions":[{"id":"https://openalex.org/I1331384533","display_name":"University of Hawaii System","ror":"https://ror.org/03tzaeb71","country_code":"US","type":"education","lineage":["https://openalex.org/I1331384533"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Italo Santos","raw_affiliation_strings":["University of Hawai‘i at Mānoa,Honolulu,Hawai\u2018i,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Hawai‘i at Mānoa,Honolulu,Hawai\u2018i,USA","institution_ids":["https://openalex.org/I1331384533"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091492434","display_name":"Brody Stuart-Verner","orcid":null},"institutions":[{"id":"https://openalex.org/I168635309","display_name":"University of Calgary","ror":"https://ror.org/03yjb2x39","country_code":"CA","type":"education","lineage":["https://openalex.org/I168635309"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Brody Stuart-Verner","raw_affiliation_strings":["University of Calgary,Calgary,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Calgary,Calgary,Canada","institution_ids":["https://openalex.org/I168635309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080161379","display_name":"Ronnie de Souza Santos","orcid":"https://orcid.org/0000-0003-3235-6530"},"institutions":[{"id":"https://openalex.org/I168635309","display_name":"University of Calgary","ror":"https://ror.org/03yjb2x39","country_code":"CA","type":"education","lineage":["https://openalex.org/I168635309"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ronnie De Souza Santos","raw_affiliation_strings":["University of Calgary,Calgary,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Calgary,Calgary,Canada","institution_ids":["https://openalex.org/I168635309"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7117,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.8717387,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"78","last_page":"88"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9031000137329102,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9031000137329102,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/unit-testing","display_name":"Unit testing","score":0.4713999927043915},{"id":"https://openalex.org/keywords/exploratory-research","display_name":"Exploratory research","score":0.46059998869895935},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.45809999108314514},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.45570001006126404},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.41449999809265137},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.4023999869823456},{"id":"https://openalex.org/keywords/system-integration-testing","display_name":"System integration testing","score":0.35929998755455017},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.35839998722076416},{"id":"https://openalex.org/keywords/system-testing","display_name":"System testing","score":0.3555999994277954}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6471999883651733},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5900999903678894},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.4713999927043915},{"id":"https://openalex.org/C85973986","wikidata":"https://www.wikidata.org/wiki/Q1091731","display_name":"Exploratory research","level":2,"score":0.46059998869895935},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.45809999108314514},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.45570001006126404},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.41449999809265137},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.4023999869823456},{"id":"https://openalex.org/C111524372","wikidata":"https://www.wikidata.org/wiki/Q7663718","display_name":"System integration testing","level":5,"score":0.35929998755455017},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.35839998722076416},{"id":"https://openalex.org/C7166840","wikidata":"https://www.wikidata.org/wiki/Q1199682","display_name":"System testing","level":2,"score":0.3555999994277954},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.34549999237060547},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.33640000224113464},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.33410000801086426},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.33079999685287476},{"id":"https://openalex.org/C107683887","wikidata":"https://www.wikidata.org/wiki/Q782466","display_name":"Integration testing","level":3,"score":0.31439998745918274},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.2978000044822693},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.29420000314712524},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.2879999876022339},{"id":"https://openalex.org/C74196892","wikidata":"https://www.wikidata.org/wiki/Q7781188","display_name":"Thematic analysis","level":3,"score":0.2849999964237213},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.28459998965263367},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.27880001068115234},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2782000005245209},{"id":"https://openalex.org/C19527686","wikidata":"https://www.wikidata.org/wiki/Q1665453","display_name":"System integration","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.26600000262260437},{"id":"https://openalex.org/C39890963","wikidata":"https://www.wikidata.org/wiki/Q1702721","display_name":"Personal software process","level":5,"score":0.26409998536109924},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26249998807907104},{"id":"https://openalex.org/C63406617","wikidata":"https://www.wikidata.org/wiki/Q5266714","display_name":"Development testing","level":5,"score":0.2574000060558319}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/scam67354.2025.00015","is_oa":false,"landing_page_url":"https://doi.org/10.1109/scam67354.2025.00015","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Source Code Analysis & Manipulation (SCAM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1969939902","https://openalex.org/W2162739315","https://openalex.org/W4391974543","https://openalex.org/W4396861092","https://openalex.org/W4396903441","https://openalex.org/W4396919238","https://openalex.org/W4401908463","https://openalex.org/W4402665833","https://openalex.org/W4402977950","https://openalex.org/W4404784286","https://openalex.org/W4405750689","https://openalex.org/W4405796591","https://openalex.org/W4406738283","https://openalex.org/W4408562727","https://openalex.org/W4409616698","https://openalex.org/W4409721409","https://openalex.org/W4409736067","https://openalex.org/W4412164158"],"related_works":[],"abstract_inverted_index":{"Background:":[0],"Software":[1],"systems":[2,52,66,159],"powered":[3],"by":[4,88,111],"large":[5],"language":[6],"models":[7],"are":[8,56,67],"becoming":[9],"a":[10,19,99,112],"routine":[11],"part":[12,97],"of":[13,22,72,98,180],"everyday":[14],"technologies,":[15],"supporting":[16],"applications":[17,95],"across":[18],"wide":[20],"range":[21],"domains.":[23],"In":[24],"software":[25,185],"engineering,":[26],"many":[27],"studies":[28],"have":[29],"focused":[30],"on":[31,49,176],"how":[32,50,64],"LLMs":[33,55],"support":[34],"tasks":[35],"such":[36],"as":[37,96],"code":[38],"generation,":[39],"debugging,":[40],"and":[41,92,121,129,139,152],"documentation.":[42],"However,":[43],"there":[44],"has":[45],"been":[46],"limited":[47],"focus":[48],"full":[51],"that":[53],"integrate":[54],"tested":[57,68],"during":[58],"development.":[59,75],"Aims:":[60],"This":[61],"study":[62,82],"explores":[63],"LLM-powered":[65,94,158],"in":[69,184],"the":[70,177],"context":[71,179],"real-world":[73],"application":[74],"Method:":[76],"We":[77],"conducted":[78],"an":[79],"exploratory":[80,135],"case":[81],"using":[83,107],"99":[84],"individual":[85],"reports":[86],"written":[87],"students":[89],"who":[90],"built":[91],"deployed":[93],"university":[100],"course.":[101],"Each":[102],"report":[103],"was":[104],"independently":[105],"analyzed":[106],"thematic":[108],"analysis,":[109],"supported":[110],"structured":[113],"coding":[114],"process.":[115],"Results:":[116],"Testing":[117,157],"strategies":[118],"combined":[119],"manual":[120],"automated":[122],"methods":[123],"to":[124,162],"evaluate":[125],"both":[126],"system":[127],"logic":[128],"model":[130],"behavior.":[131],"Common":[132],"practices":[133],"included":[134,144],"testing,":[136,138],"unit":[137],"prompt":[140,149],"iteration.":[141],"Reported":[142],"challenges":[143],"integration":[145],"failures,":[146],"unpredictable":[147],"outputs,":[148],"sensitivity,":[150],"hallucinations,":[151],"uncertainty":[153],"about":[154],"correctness.":[155],"Conclusions:":[156],"required":[160],"adaptations":[161],"traditional":[163],"verification":[164],"methods,":[165],"blending":[166],"source-level":[167],"reasoning":[168],"with":[169],"behavior-aware":[170],"evaluations.":[171],"These":[172],"findings":[173],"provide":[174],"evidence":[175],"practical":[178],"testing":[181],"generative":[182],"components":[183],"systems.":[186]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}