{"id":"https://openalex.org/W7125943639","doi":"https://doi.org/10.1109/ase63991.2025.00383","title":"Quirx: A Mutation-Based Framework for Evaluating Prompt Robustness in LLM-based Software","display_name":"Quirx: A Mutation-Based Framework for Evaluating Prompt Robustness in LLM-based Software","publication_year":2025,"publication_date":"2025-11-16","ids":{"openalex":"https://openalex.org/W7125943639","doi":"https://doi.org/10.1109/ase63991.2025.00383"},"language":null,"primary_location":{"id":"doi:10.1109/ase63991.2025.00383","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ase63991.2025.00383","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 40th IEEE/ACM International Conference on Automated Software Engineering (ASE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020388180","display_name":"Souhaila Serbout","orcid":"https://orcid.org/0000-0002-8144-2606"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Souhaila Serbout","raw_affiliation_strings":["University of Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zurich","institution_ids":["https://openalex.org/I202697423"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5020388180"],"corresponding_institution_ids":["https://openalex.org/I202697423"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.75069845,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4094","last_page":"4097"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.3506999909877777,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.3506999909877777,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.13369999825954437,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.07840000092983246,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7457000017166138},{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.6097999811172485},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5842000246047974},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.42160001397132874},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4081000089645386},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.40070000290870667}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7457000017166138},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7246999740600586},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.6097999811172485},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5842000246047974},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5013999938964844},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.42160001397132874},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4081000089645386},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.40070000290870667},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.39010000228881836},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37279999256134033},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3693000078201294},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.3643999993801117},{"id":"https://openalex.org/C2776825360","wikidata":"https://www.wikidata.org/wiki/Q1411921","display_name":"Vagueness","level":3,"score":0.3531999886035919},{"id":"https://openalex.org/C111065885","wikidata":"https://www.wikidata.org/wiki/Q1189053","display_name":"Fuzz testing","level":3,"score":0.3382999897003174},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.3174999952316284},{"id":"https://openalex.org/C510870499","wikidata":"https://www.wikidata.org/wiki/Q47607","display_name":"SQL","level":2,"score":0.2736999988555908},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.25850000977516174}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ase63991.2025.00383","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ase63991.2025.00383","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 40th IEEE/ACM International Conference on Automated Software Engineering (ASE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.43501007556915283}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W2806377938","https://openalex.org/W2970641574","https://openalex.org/W3105604018","https://openalex.org/W3160638507","https://openalex.org/W3185341429","https://openalex.org/W4415798641"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"increasingly":[4],"power":[5],"critical":[6],"business":[7],"processes,":[8],"yet":[9],"prompt":[10,56],"robustness":[11,57,83],"remains":[12],"under-explored.":[13],"Small":[14],"variations\u2014such":[15],"as":[16],"synonym":[17],"changes":[18],"or":[19],"instruction":[20],"reordering\u2014can":[21],"cause":[22,135],"significant":[23],"output":[24],"shifts,":[25],"undermining":[26],"reliability":[27],"in":[28,44],"domains":[29],"like":[30],"customer":[31],"service":[32],"and":[33,73,88,126],"finance.":[34],"Existing":[35],"evaluations":[36],"rely":[37],"on":[38,95,144],"ad-hoc":[39],"manual":[40],"testing,":[41],"limiting":[42],"scalability":[43],"production":[45],"environments.We":[46],"present":[47],"Quirx,":[48],"a":[49],"mutation-based":[50],"fuzzing":[51],"framework":[52],"for":[53],"systematically":[54],"evaluating":[55],"across":[58,102],"LLM":[59],"providers.":[60],"Quirx":[61,94],"applies":[62],"tri-dimensional":[63],"mutations":[64,134],"(lexical,":[65],"semantic,":[66],"structural),":[67],"executes":[68],"them":[69],"against":[70],"target":[71],"models,":[72],"measures":[74],"response":[75],"consistency":[76],"via":[77],"multi-level":[78],"similarity":[79],"analysis.":[80],"It":[81],"produces":[82],"scores,":[84],"reveals":[85],"failure":[86],"patterns,":[87],"supports":[89],"informed":[90],"model":[91],"selection.We":[92],"evaluate":[93],"four":[96],"models":[97,120],"(GPT-3.5-turbo,":[98],"GPT-4o-mini,":[99],"Claude-3.5-Sonnet,":[100],"Claude-Sonnet-4)":[101],"three":[103],"tasks.":[104],"Results":[105],"show":[106],"sentiment":[107],"classification":[108],"is":[109,114,129],"uniformly":[110],"robust":[111,123],"(1.00),":[112],"summarization":[113,138],"highly":[115],"provider-sensitive":[116],"(0.23\u20130.58)":[117],"with":[118],"Claude":[119],"2.5\u00d7":[121],"more":[122],"than":[124],"OpenAI,":[125],"SQL":[127],"generation":[128],"consistently":[130],"strong":[131],"(0.80\u20131.00).":[132],"Structural":[133],"50\u201367%":[136],"of":[137],"failures":[139],"but":[140],"have":[141],"minimal":[142],"effect":[143],"other":[145],"tasks.Demo":[146],"video:":[147],"https://youtu.be/Sm3Gk2X2-vk":[148]},"counts_by_year":[],"updated_date":"2026-01-29T23:17:01.242718","created_date":"2026-01-29T00:00:00"}