{"id":"https://openalex.org/W4409495145","doi":"https://doi.org/10.1109/icstw64639.2025.10962487","title":"Quality Assurance for LLM-RAG Systems: Empirical Insights from Tourism Application Testing","display_name":"Quality Assurance for LLM-RAG Systems: Empirical Insights from Tourism Application Testing","publication_year":2025,"publication_date":"2025-03-31","ids":{"openalex":"https://openalex.org/W4409495145","doi":"https://doi.org/10.1109/icstw64639.2025.10962487"},"language":"en","primary_location":{"id":"doi:10.1109/icstw64639.2025.10962487","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icstw64639.2025.10962487","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031391282","display_name":"Bestoun S. Ahmed","orcid":"https://orcid.org/0000-0001-9051-7609"},"institutions":[{"id":"https://openalex.org/I106663880","display_name":"University of Bahrain","ror":"https://ror.org/0317ekv86","country_code":"BH","type":"education","lineage":["https://openalex.org/I106663880"]},{"id":"https://openalex.org/I300940544","display_name":"University of Technology Bahrain","ror":"https://ror.org/00h15ta02","country_code":"BH","type":"education","lineage":["https://openalex.org/I300940544"]},{"id":"https://openalex.org/I4387155350","display_name":"American University of Bahrain","ror":"https://ror.org/00b8p9q06","country_code":"BH","type":"education","lineage":["https://openalex.org/I4387155350"]},{"id":"https://openalex.org/I43968019","display_name":"Karlstad University","ror":"https://ror.org/05s754026","country_code":"SE","type":"education","lineage":["https://openalex.org/I43968019"]}],"countries":["BH","SE"],"is_corresponding":false,"raw_author_name":"Bestoun S. Ahmed","raw_affiliation_strings":["Karlstad University, Sweden & American University of Bahrain,Dept. Mathematics and Computer Science,Bahrain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Karlstad University, Sweden & American University of Bahrain,Dept. Mathematics and Computer Science,Bahrain","institution_ids":["https://openalex.org/I106663880","https://openalex.org/I300940544","https://openalex.org/I4387155350","https://openalex.org/I43968019"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116235537","display_name":"Ludwig Otto Baader","orcid":null},"institutions":[{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ludwig Otto Baader","raw_affiliation_strings":["Ludwig Maximilian University Munich,Dept. Mathematics, Informatics and Statistics,Munich,Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ludwig Maximilian University Munich,Dept. Mathematics, Informatics and Statistics,Munich,Germany","institution_ids":["https://openalex.org/I8204097"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024836775","display_name":"Firas Bayram","orcid":"https://orcid.org/0000-0003-0683-2783"},"institutions":[{"id":"https://openalex.org/I43968019","display_name":"Karlstad University","ror":"https://ror.org/05s754026","country_code":"SE","type":"education","lineage":["https://openalex.org/I43968019"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Firas Bayram","raw_affiliation_strings":["Karlstad University,Dept. Mathematics and Computer Science,Karlstad,Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Karlstad University,Dept. Mathematics and Computer Science,Karlstad,Sweden","institution_ids":["https://openalex.org/I43968019"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011319459","display_name":"Siri Jagstedt","orcid":"https://orcid.org/0000-0002-3281-7942"},"institutions":[{"id":"https://openalex.org/I43968019","display_name":"Karlstad University","ror":"https://ror.org/05s754026","country_code":"SE","type":"education","lineage":["https://openalex.org/I43968019"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Siri Jagstedt","raw_affiliation_strings":["Karlstad University,CTF, Service Research Center,Karlstad,Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Karlstad University,CTF, Service Research Center,Karlstad,Sweden","institution_ids":["https://openalex.org/I43968019"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048642325","display_name":"Peter Magnusson","orcid":"https://orcid.org/0000-0002-7653-5226"},"institutions":[{"id":"https://openalex.org/I43968019","display_name":"Karlstad University","ror":"https://ror.org/05s754026","country_code":"SE","type":"education","lineage":["https://openalex.org/I43968019"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Peter Magnusson","raw_affiliation_strings":["Karlstad University,CTF, Service Research Center,Karlstad,Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Karlstad University,CTF, Service Research Center,Karlstad,Sweden","institution_ids":["https://openalex.org/I43968019"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":18.4032,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.99386313,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"200","last_page":"207"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.5099999904632568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.5099999904632568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.4805000126361847,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.4521999955177307,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quality-assurance","display_name":"Quality assurance","score":0.6605398654937744},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.576825737953186},{"id":"https://openalex.org/keywords/tourism","display_name":"Tourism","score":0.5589668154716492},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4700412154197693},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.440568208694458},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.3543488383293152},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.2688634395599365},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.1056239902973175}],"concepts":[{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.6605398654937744},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.576825737953186},{"id":"https://openalex.org/C18918823","wikidata":"https://www.wikidata.org/wiki/Q49389","display_name":"Tourism","level":2,"score":0.5589668154716492},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4700412154197693},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.440568208694458},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3543488383293152},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2688634395599365},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.1056239902973175},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icstw64639.2025.10962487","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icstw64639.2025.10962487","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6899999976158142,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W4309043734","https://openalex.org/W4391136507","https://openalex.org/W4391974543","https://openalex.org/W4402670429","https://openalex.org/W6777615688","https://openalex.org/W6778883912","https://openalex.org/W6857079699","https://openalex.org/W6861295083","https://openalex.org/W6866678611","https://openalex.org/W6869771570","https://openalex.org/W6874524037"],"related_works":["https://openalex.org/W2391061603","https://openalex.org/W2386798499","https://openalex.org/W4388017739","https://openalex.org/W2352379149","https://openalex.org/W3216130691","https://openalex.org/W2369032308","https://openalex.org/W2977338326","https://openalex.org/W2376872419","https://openalex.org/W11269544","https://openalex.org/W4389096689"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"a":[3,107],"comprehensive":[4],"framework":[5,56],"for":[6,111,157],"testing":[7,45,160],"and":[8,52,67,83,95,119,145],"evaluating":[9],"quality":[10],"characteristics":[11],"of":[12,30,43,93],"Large":[13],"Language":[14],"Model":[15],"(LLM)":[16],"systems":[17],"enhanced":[18],"with":[19],"Retrieval-Augmented":[20],"Generation":[21],"(RAG)":[22],"in":[23,47,115,133,142,149,162,173],"tourism":[24,108],"applications.":[25],"Through":[26],"systematic":[27],"empirical":[28],"evaluation":[29,69],"three":[31],"different":[32,80],"LLM":[33,71,128],"variants":[34],"across":[35],"multiple":[36],"parameter":[37,84],"configurations,":[38],"we":[39],"demonstrate":[40],"the":[41,91,112,126,137],"effectiveness":[42],"our":[44],"methodology":[46],"assessing":[48],"both":[49],"functional":[50],"correctness":[51],"extra-functional":[53],"properties.":[54],"Our":[55],"implements":[57],"17":[58],"distinct":[59],"metrics":[60],"that":[61,125],"encompass":[62],"syntactic":[63],"analysis,":[64],"semantic":[65,150],"evaluation,":[66],"behavioral":[68],"through":[70],"judges.":[72],"The":[73,101,122,152],"study":[74],"reveals":[75],"significant":[76],"information":[77],"about":[78],"how":[79],"architectural":[81],"choices":[82],"configurations":[85],"affect":[86],"system":[87,110],"performance,":[88],"particularly":[89],"highlighting":[90],"impact":[92],"temperature":[94],"top-p":[96],"parameters":[97],"on":[98,106],"response":[99,143],"quality.":[100,151],"tests":[102],"were":[103],"carried":[104],"out":[105],"recommendation":[109],"Varmland":[113],"region":[114],"Sweden,":[116],"utilizing":[117],"standard":[118],"RAG-enhanced":[120],"configurations.":[121],"results":[123],"indicate":[124],"newer":[127],"versions":[129],"show":[130],"modest":[131],"improvements":[132],"performance":[134],"metrics,":[135],"though":[136],"differences":[138],"are":[139],"more":[140],"pronounced":[141],"length":[144],"complexity":[146],"rather":[147],"than":[148],"research":[153],"contributes":[154],"practical":[155],"insights":[156],"implementing":[158],"robust":[159],"practices":[161],"LLM-RAG":[163],"systems,":[164],"providing":[165],"valuable":[166],"guidance":[167],"to":[168],"organizations":[169],"deploying":[170],"these":[171],"architectures":[172],"production":[174],"environments.":[175]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":6}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}