{"id":"https://openalex.org/W4416254755","doi":"https://doi.org/10.5220/0014980700004015","title":"On the Soundness and Consistency of LLM Agents for Executing Test Cases Written in Natural Language","display_name":"On the Soundness and Consistency of LLM Agents for Executing Test Cases Written in Natural Language","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W4416254755","doi":"https://doi.org/10.5220/0014980700004015"},"language":"en","primary_location":{"id":"doi:10.5220/0014980700004015","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0014980700004015","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Evaluation of Novel Approaches to Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.5220/0014980700004015","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032624369","display_name":"S\u00e9bastien Salva","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I4210099416","display_name":"Laboratoire d'Informatique, de Mod\u00e9lisation et d'Optimisation des Syst\u00e8mes","ror":"https://ror.org/00t3fpp34","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I198244214","https://openalex.org/I198244214","https://openalex.org/I203339264","https://openalex.org/I205703379","https://openalex.org/I3019848993","https://openalex.org/I4210099416","https://openalex.org/I4210123221","https://openalex.org/I4210159245","https://openalex.org/I4387154249"]},{"id":"https://openalex.org/I4210103002","display_name":"University of Clermont Auvergne","ror":null,"country_code":"FR","type":null,"lineage":["https://openalex.org/I4210103002"]},{"id":"https://openalex.org/I4210143836","display_name":"Clermont Universit\u00e9","ror":"https://ror.org/03yf5zr20","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210143836"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"S\u00e9bastien Salva","raw_affiliation_strings":["IUT CA, Clermont Auvergne University, UCA, Aubi\u00e8re, France","LIMOS - UMR CNRS 6158, Aubi\u00e8re, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IUT CA, Clermont Auvergne University, UCA, Aubi\u00e8re, France","institution_ids":["https://openalex.org/I4210103002","https://openalex.org/I4210143836"]},{"raw_affiliation_string":"LIMOS - UMR CNRS 6158, Aubi\u00e8re, France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I4210099416"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021856906","display_name":"Redha Taguelmimt","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I4210099416","display_name":"Laboratoire d'Informatique, de Mod\u00e9lisation et d'Optimisation des Syst\u00e8mes","ror":"https://ror.org/00t3fpp34","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I198244214","https://openalex.org/I198244214","https://openalex.org/I203339264","https://openalex.org/I205703379","https://openalex.org/I3019848993","https://openalex.org/I4210099416","https://openalex.org/I4210123221","https://openalex.org/I4210159245","https://openalex.org/I4387154249"]},{"id":"https://openalex.org/I4210103002","display_name":"University of Clermont Auvergne","ror":null,"country_code":"FR","type":null,"lineage":["https://openalex.org/I4210103002"]},{"id":"https://openalex.org/I4210143836","display_name":"Clermont Universit\u00e9","ror":"https://ror.org/03yf5zr20","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210143836"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Redha Taguelmimt","raw_affiliation_strings":["IUT CA, Clermont Auvergne University, UCA, Aubi\u00e8re, France","LIMOS - UMR CNRS 6158, Aubi\u00e8re, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IUT CA, Clermont Auvergne University, UCA, Aubi\u00e8re, France","institution_ids":["https://openalex.org/I4210103002","https://openalex.org/I4210143836"]},{"raw_affiliation_string":"LIMOS - UMR CNRS 6158, Aubi\u00e8re, France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I4210099416"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02146465,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1007","last_page":"1018"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5781000256538391,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5781000256538391,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0632999986410141,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.053700000047683716,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.76419997215271},{"id":"https://openalex.org/keywords/soundness","display_name":"Soundness","score":0.7229999899864197},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.7032999992370605},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.670199990272522},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.545799970626831},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.41190001368522644},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.40549999475479126},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.3946000039577484}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8080000281333923},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.76419997215271},{"id":"https://openalex.org/C39920170","wikidata":"https://www.wikidata.org/wiki/Q693083","display_name":"Soundness","level":2,"score":0.7229999899864197},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7032999992370605},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.670199990272522},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6014000177383423},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.545799970626831},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.41190001368522644},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.3946000039577484},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C109852812","wikidata":"https://www.wikidata.org/wiki/Q2406355","display_name":"Test harness","level":5,"score":0.3476000130176544},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.3425999879837036},{"id":"https://openalex.org/C109086967","wikidata":"https://www.wikidata.org/wiki/Q2509100","display_name":"Test script","level":4,"score":0.29679998755455017},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2944999933242798},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.28870001435279846},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2705000042915344},{"id":"https://openalex.org/C66153210","wikidata":"https://www.wikidata.org/wiki/Q5597182","display_name":"Graphical user interface testing","level":4,"score":0.2678000032901764},{"id":"https://openalex.org/C7435765","wikidata":"https://www.wikidata.org/wiki/Q7705776","display_name":"Test Management Approach","level":5,"score":0.2653000056743622},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.2565000057220459}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.5220/0014980700004015","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0014980700004015","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Evaluation of Novel Approaches to Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2509.19136","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.19136","pdf_url":"https://arxiv.org/pdf/2509.19136","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-05672526v1","is_oa":false,"landing_page_url":"https://uca.hal.science/hal-05672526","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://www.scitepress.org/ProceedingsDetails.aspx?ID=blsLvqpiJl8=&t=1","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"doi:10.48550/arxiv.2509.19136","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.19136","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5220/0014980700004015","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0014980700004015","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Evaluation of Novel Approaches to Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"use":[1],"of":[2,47,51,101,140,151,167,211],"natural":[3],"language":[4,40],"(NL)":[5],"test":[6,25,53,69,74,79,105,113,126,142,154,176,231,246],"cases":[7,54,80,127],"for":[8,123,215],"validating":[9],"graphical":[10],"user":[11],"interface":[12],"(GUI)":[13],"applications":[14],"is":[15],"emerging":[16],"as":[17,84],"a":[18,165],"promising":[19],"direction":[20],"to":[21,30,34,91,109,147,159,170,183,201],"manually":[22],"written":[23],"executable":[24],"scripts,":[26],"which":[27,174],"are":[28,81],"costly":[29],"develop":[31],"and":[32,72,131,156,208,248],"difficult":[33],"maintain.":[35],"Recent":[36],"advances":[37],"in":[38,153,173,229],"large":[39],"models":[41],"(LLMs)":[42],"have":[43],"opened":[44],"the":[45,48,65,102,137,149,184,206,239],"possibility":[46],"direct":[49],"execution":[50,76,139,155,161,178,233,236],"NL":[52,68,78,104,125,175,230],"by":[55],"LLM":[56,213],"agents.":[57],"This":[58],"paper":[59],"investigates":[60],"this":[61],"direction,":[62],"focusing":[63],"on":[64,67,73],"impact":[66],"case":[70,75,106,177,232],"unsoundness":[71,169],"consistency.":[77,162],"inherently":[82],"unsound,":[83],"they":[85],"may":[86,107],"yield":[87],"false":[88],"failures":[89],"due":[90],"ambiguous":[92],"instructions":[93],"or":[94],"unpredictable":[95],"agent":[96],"behaviour.":[97],"Furthermore,":[98],"repeated":[99],"executions":[100],"same":[103],"lead":[108],"inconsistent":[110],"outcomes,":[111],"undermining":[112],"reliability.":[114],"To":[115],"address":[116],"these":[117],"challenges,":[118],"we":[119],"propose":[120,164],"an":[121],"algorithm":[122],"executing":[124],"with":[128,181,193,234],"guardrail":[129],"mechanisms":[130],"specialised":[132],"agents":[133,214],"that":[134,221],"dynamically":[135],"verify":[136],"correct":[138],"each":[141],"step.":[143],"We":[144,163,242],"introduce":[145],"measures":[146],"evaluate":[148],"capabilities":[150,228],"LLMs":[152],"one":[157],"measure":[158],"quantify":[160],"definition":[166],"weak":[168],"characterise":[171],"contexts":[172],"remains":[179],"acceptable,":[180],"respect":[182],"industrial":[185],"quality":[186],"levels":[187],"Six":[188],"Sigma.":[189],"Our":[190,218],"experimental":[191],"evaluation":[192],"eight":[194],"publicly":[195],"available":[196],"LLMs,":[197],"ranging":[198],"from":[199],"3B":[200],"70B":[202,225],"parameters,":[203],"demonstrates":[204,226],"both":[205],"potential":[207],"current":[209,212],"limitations":[210],"GUI":[216],"testing.":[217],"experiments":[219],"show":[220],"Meta":[222],"Llama":[223],"3.1":[224],"acceptable":[227],"high":[235],"consistency":[237],"(above":[238],"level":[240],"3-sigma).":[241],"provide":[243],"prototype":[244],"tools,":[245],"suites,":[247],"results.":[249]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}