{"id":"https://openalex.org/W7155366293","doi":"https://doi.org/10.1109/ats66998.2025.00014","title":"Exploring the Limits of LLMs for System-Level Test Program Generation: Can LLaMas Outrun Darwin?","display_name":"Exploring the Limits of LLMs for System-Level Test Program Generation: Can LLaMas Outrun Darwin?","publication_year":2025,"publication_date":"2025-12-16","ids":{"openalex":"https://openalex.org/W7155366293","doi":"https://doi.org/10.1109/ats66998.2025.00014"},"language":null,"primary_location":{"id":"doi:10.1109/ats66998.2025.00014","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ats66998.2025.00014","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 34th Asian Test Symposium (ATS)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068934998","display_name":"Denis Schwachhofer","orcid":"https://orcid.org/0000-0002-6763-2948"},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Denis Schwachhofer","raw_affiliation_strings":["Institute of Software Engineering University of Stuttgart,Stuttgart,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software Engineering University of Stuttgart,Stuttgart,Germany","institution_ids":["https://openalex.org/I100066346"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054632200","display_name":"Steffen Becker","orcid":"https://orcid.org/0000-0002-4532-1460"},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Steffen Becker","raw_affiliation_strings":["Institute of Software Engineering University of Stuttgart,Stuttgart,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software Engineering University of Stuttgart,Stuttgart,Germany","institution_ids":["https://openalex.org/I100066346"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134374886","display_name":"Stefan Wagner","orcid":null},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Stefan Wagner","raw_affiliation_strings":["Institute of Software Engineering University of Stuttgart,Stuttgart,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software Engineering University of Stuttgart,Stuttgart,Germany","institution_ids":["https://openalex.org/I100066346"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111203264","display_name":"Matthias Sauer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matthias Sauer","raw_affiliation_strings":["Advantest Europe,Boeblingen,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Advantest Europe,Boeblingen,Germany","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5134439811","display_name":"Ilia Polian","orcid":null},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ilia Polian","raw_affiliation_strings":["Institute of Computer Engineering and Computer Architecture University of Stuttgart,Stuttgart,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computer Engineering and Computer Architecture University of Stuttgart,Stuttgart,Germany","institution_ids":["https://openalex.org/I100066346"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.77375025,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"31","last_page":"36"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.46320000290870667,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.46320000290870667,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.09189999848604202,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13293","display_name":"Engineering and Test Systems","score":0.05460000038146973,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5333999991416931},{"id":"https://openalex.org/keywords/ethnic-group","display_name":"Ethnic group","score":0.2443999946117401},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.24009999632835388}],"concepts":[{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5333999991416931},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.41679999232292175},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.3806999921798706},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.3100999891757965},{"id":"https://openalex.org/C47768531","wikidata":"https://www.wikidata.org/wiki/Q1127188","display_name":"Development economics","level":1,"score":0.2833999991416931},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.2799000144004822},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.265500009059906},{"id":"https://openalex.org/C3116431","wikidata":"https://www.wikidata.org/wiki/Q31728","display_name":"Public administration","level":1,"score":0.24869999289512634},{"id":"https://openalex.org/C137403100","wikidata":"https://www.wikidata.org/wiki/Q41710","display_name":"Ethnic group","level":2,"score":0.2443999946117401},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.24009999632835388}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ats66998.2025.00014","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ats66998.2025.00014","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 34th Asian Test Symposium (ATS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"System-Level":[0],"Test":[1,17],"(SLT)":[2],"is":[3],"important":[4],"in":[5,29,67,77,163],"semiconductor":[6],"testing":[7],"as":[8,33],"it":[9],"can":[10,74],"detect":[11],"defects":[12],"missed":[13],"by":[14],"traditional":[15],"methods.":[16],"engineers":[18],"use":[19],"off-the-shelf":[20],"software":[21],"to":[22,81,108,122],"manually":[23],"compose":[24],"test":[25,44,59],"suites,":[26],"often":[27],"written":[28],"high-level":[30,78,94],"languages":[31,79,95],"such":[32],"$\\mathrm{C}":[34],"/":[35],"\\mathrm{C}++$":[36],"or":[37],"Rust.":[38],"Several":[39],"methods":[40],"for":[41,96,179],"automatically":[42],"generating":[43,97],"programs":[45,60],"have":[46,138,172],"been":[47],"investigated,":[48],"using":[49,104],"assembly":[50,111],"language.":[51],"However,":[52],"one":[53],"could":[54],"argue":[55],"that":[56,128,152,168],"the":[57,89,114,129,131,136,143,153,156,169,181,184],"resulting":[58,144],"are":[61],"not":[62],"capturing":[63],"all":[64],"possible":[65],"interactions":[66],"actual":[68,82],"software.":[69,83],"Large":[70],"Language":[71],"Models":[72],"(LLMs)":[73],"generate":[75,123],"code":[76,125],"closer":[80],"In":[84],"this":[85],"work,":[86],"we":[87,119,148,166],"examine":[88],"limitations":[90],"of":[91],"LLMs":[92,121],"and":[93,126,135,183],"SLT":[98],"programs.":[99],"We":[100],"run":[101,158],"an":[102,110],"experiment":[103],"genetic":[105],"programming":[106],"(GP)":[107],"find":[109],"snippet":[112,154],"with":[113],"highest":[115],"power":[116,145,177],"consumption.":[117,146],"Then,":[118],"utilize":[120],"$\\mathbf{C}$":[124],"demonstrate":[127,167],"compiler,":[130],"enabled":[132],"optimization":[133],"level,":[134],"LLM":[137],"a":[139,173],"significant":[140,174],"influence":[141],"on":[142,176],"Furthermore,":[147],"show":[149],"via":[150],"decompilation":[151],"from":[155],"GP":[157],"has":[159],"no":[160],"direct":[161],"equivalent":[162],"C.":[164],"Finally,":[165],"initial":[170],"values":[171],"impact":[175],"consumption":[178],"both":[180],"GP-generated":[182],"decompiled":[185],"snippet.":[186]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-04-24T00:00:00"}