{"id":"https://openalex.org/W7162552862","doi":"https://doi.org/10.1016/j.iswa.2026.200681","title":"RAG-Augmented LLMs for Penetration Testing: A benchmarking study of open-source LLM models","display_name":"RAG-Augmented LLMs for Penetration Testing: A benchmarking study of open-source LLM models","publication_year":2026,"publication_date":"2026-05-27","ids":{"openalex":"https://openalex.org/W7162552862","doi":"https://doi.org/10.1016/j.iswa.2026.200681"},"language":"en","primary_location":{"id":"doi:10.1016/j.iswa.2026.200681","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.iswa.2026.200681","pdf_url":null,"source":{"id":"https://openalex.org/S4210234522","display_name":"Intelligent Systems with Applications","issn_l":"2667-3053","issn":["2667-3053"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Intelligent Systems with Applications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1016/j.iswa.2026.200681","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137138332","display_name":"Oumaima Ben Fadhel","orcid":null},"institutions":[{"id":"https://openalex.org/I185808892","display_name":"Polytechnic Institute of C\u00e1vado and Ave","ror":"https://ror.org/0448qsq10","country_code":"PT","type":"education","lineage":["https://openalex.org/I185808892"]},{"id":"https://openalex.org/I4210088638","display_name":"Higher Institute of Management","ror":"https://ror.org/003sxhp42","country_code":"RU","type":"education","lineage":["https://openalex.org/I4210088638"]}],"countries":["PT","RU"],"is_corresponding":false,"raw_author_name":"Oumaima Ben Fadhel","raw_affiliation_strings":["2Ai - School of Technology, IPCA, Barcelos, Portugal","ISI - Higher Institute of Computer Science, Tunis, Tunisia","LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"2Ai - School of Technology, IPCA, Barcelos, Portugal","institution_ids":["https://openalex.org/I185808892"]},{"raw_affiliation_string":"ISI - Higher Institute of Computer Science, Tunis, Tunisia","institution_ids":["https://openalex.org/I4210088638"]},{"raw_affiliation_string":"LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137123905","display_name":"Rui Fernandes","orcid":null},"institutions":[{"id":"https://openalex.org/I185808892","display_name":"Polytechnic Institute of C\u00e1vado and Ave","ror":"https://ror.org/0448qsq10","country_code":"PT","type":"education","lineage":["https://openalex.org/I185808892"]},{"id":"https://openalex.org/I4210100923","display_name":"Munster Technological University","ror":"https://ror.org/013xpqh61","country_code":"IE","type":"facility","lineage":["https://openalex.org/I4210100923"]}],"countries":["IE","PT"],"is_corresponding":true,"raw_author_name":"Rui Fernandes","raw_affiliation_strings":["2Ai - School of Technology, IPCA, Barcelos, Portugal","LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal","TUS - Technological University of the Shannon, Limerick, Ireland"],"raw_orcid":"https://orcid.org/0009-0002-1365-0208","affiliations":[{"raw_affiliation_string":"2Ai - School of Technology, IPCA, Barcelos, Portugal","institution_ids":["https://openalex.org/I185808892"]},{"raw_affiliation_string":"LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal","institution_ids":[]},{"raw_affiliation_string":"TUS - Technological University of the Shannon, Limerick, Ireland","institution_ids":["https://openalex.org/I4210100923"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073839766","display_name":"Monia Najjar","orcid":"https://orcid.org/0000-0003-4484-4523"},"institutions":[{"id":"https://openalex.org/I185808892","display_name":"Polytechnic Institute of C\u00e1vado and Ave","ror":"https://ror.org/0448qsq10","country_code":"PT","type":"education","lineage":["https://openalex.org/I185808892"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Oscar Ribeiro","raw_affiliation_strings":["2Ai - School of Technology, IPCA, Barcelos, Portugal","LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"2Ai - School of Technology, IPCA, Barcelos, Portugal","institution_ids":["https://openalex.org/I185808892"]},{"raw_affiliation_string":"LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137101098","display_name":"Oscar Ribeiro","orcid":null},"institutions":[{"id":"https://openalex.org/I4210088638","display_name":"Higher Institute of Management","ror":"https://ror.org/003sxhp42","country_code":"RU","type":"education","lineage":["https://openalex.org/I4210088638"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Monia Najjar","raw_affiliation_strings":["ISI - Higher Institute of Computer Science, Tunis, Tunisia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISI - Higher Institute of Computer Science, Tunis, Tunisia","institution_ids":["https://openalex.org/I4210088638"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5137133740","display_name":"Nuno Lopes","orcid":null},"institutions":[{"id":"https://openalex.org/I185808892","display_name":"Polytechnic Institute of C\u00e1vado and Ave","ror":"https://ror.org/0448qsq10","country_code":"PT","type":"education","lineage":["https://openalex.org/I185808892"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Nuno Lopes","raw_affiliation_strings":["2Ai - School of Technology, IPCA, Barcelos, Portugal","LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"2Ai - School of Technology, IPCA, Barcelos, Portugal","institution_ids":["https://openalex.org/I185808892"]},{"raw_affiliation_string":"LASI \u2013 Associate Laboratory of Intelligent Systems, Guimar\u00e3es, Portugal","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5137123905"],"corresponding_institution_ids":["https://openalex.org/I185808892","https://openalex.org/I4210100923"],"apc_list":{"value":1500,"currency":"USD","value_usd":1500},"apc_paid":{"value":1500,"currency":"USD","value_usd":1500},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.77959502,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"31","issue":null,"first_page":"200681","last_page":"200681"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.1046999990940094,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.1046999990940094,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.0640999972820282,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.05339999869465828,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6031000018119812},{"id":"https://openalex.org/keywords/penetration","display_name":"Penetration (warfare)","score":0.483599990606308},{"id":"https://openalex.org/keywords/risk-assessment","display_name":"Risk assessment","score":0.288100004196167}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6031000018119812},{"id":"https://openalex.org/C80107235","wikidata":"https://www.wikidata.org/wiki/Q7162625","display_name":"Penetration (warfare)","level":2,"score":0.483599990606308},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.37630000710487366},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.35580000281333923},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2924000024795532},{"id":"https://openalex.org/C91375879","wikidata":"https://www.wikidata.org/wiki/Q15473274","display_name":"Environmental planning","level":1,"score":0.289900004863739},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.288100004196167},{"id":"https://openalex.org/C77595967","wikidata":"https://www.wikidata.org/wiki/Q3151013","display_name":"Forensic engineering","level":1,"score":0.25440001487731934},{"id":"https://openalex.org/C175605778","wikidata":"https://www.wikidata.org/wiki/Q3299701","display_name":"Natural resource economics","level":1,"score":0.24979999661445618},{"id":"https://openalex.org/C39432304","wikidata":"https://www.wikidata.org/wiki/Q188847","display_name":"Environmental science","level":0,"score":0.24889999628067017}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1016/j.iswa.2026.200681","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.iswa.2026.200681","pdf_url":null,"source":{"id":"https://openalex.org/S4210234522","display_name":"Intelligent Systems with Applications","issn_l":"2667-3053","issn":["2667-3053"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Intelligent Systems with Applications","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:77df6bad4717481a9681635e20355547","is_oa":true,"landing_page_url":"https://doaj.org/article/77df6bad4717481a9681635e20355547","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Intelligent Systems with Applications, Vol 31, Iss , Pp 200681- (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1016/j.iswa.2026.200681","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.iswa.2026.200681","pdf_url":null,"source":{"id":"https://openalex.org/S4210234522","display_name":"Intelligent Systems with Applications","issn_l":"2667-3053","issn":["2667-3053"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Intelligent Systems with Applications","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334779","display_name":"Funda\u00e7\u00e3o para a Ci\u00eancia e a Tecnologia","ror":"https://ror.org/00snfqn58"},{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W4404835216","https://openalex.org/W4407163436","https://openalex.org/W4409230931","https://openalex.org/W7140203527","https://openalex.org/W7160448046"],"related_works":[],"abstract_inverted_index":{"Penetration":[0,34,53,89,133,280],"Testing":[1,35,54,90,134,281],"(PT),":[2],"also":[3],"referred":[4],"to":[5,25,33,48,74,84,96,152,209,273],"as":[6,76],"Ethical":[7],"Hacking,":[8],"is":[9,82,141],"a":[10,59,66,137],"fundamental":[11],"practice":[12],"in":[13,107,156],"Cybersecurity":[14,227],"that":[15,64,234,262],"involves":[16],"evaluating":[17],"the":[18,49,88,112,115,118,168,219,242,259],"security":[19],"of":[20,52,117],"systems,":[21],"networks,":[22],"and":[23,28,43,104,129,176,188,194,231,238],"applications":[24],"identify":[26],"vulnerabilities":[27],"potential":[29],"attacks.":[30],"Traditional":[31],"approaches":[32],"are":[36],"often":[37],"time-consuming,":[38],"relying":[39],"on":[40,245],"manual":[41],"processes,":[42],"requiring":[44],"advanced":[45],"expertise":[46],"due":[47],"technical":[50,105,186],"complexity":[51],"tools.":[55],"In":[56],"this":[57],"paper,":[58],"Retrieval-Augmented":[60],"Generation":[61],"(RAG)":[62],"pipeline":[63,120],"enhances":[65],"trained":[67],"Large":[68],"Language":[69],"Model":[70],"(LLM)":[71],"was":[72],"developed":[73],"serve":[75],"an":[77],"intelligent":[78],"Pentest":[79],"Assistant.":[80],"It":[81],"designed":[83],"assist":[85],"users":[86],"throughout":[87],"workflow,":[91],"ranging":[92],"from":[93],"vulnerability":[94],"assessment":[95],"remediation,":[97],"by":[98],"providing":[99],"contextual":[100,127],"guidance,":[101,190],"command":[102],"suggestions,":[103],"explanations":[106],"natural":[108],"language.":[109],"After":[110],"implementing":[111],"proposed":[113],"architecture,":[114],"integration":[116],"RAG":[119],"with":[121,184,252,267],"LLMs":[122,146,272],"demonstrably":[123],"enhanced":[124],"response":[125],"accuracy,":[126],"relevance,":[128],"domain":[130],"alignment":[131],"across":[132,143,278],"tasks.":[135],"Additionally,":[136],"systematic":[138],"benchmarking":[139,161],"study":[140],"conducted":[142],"seven":[144],"open-source":[145,271],"under":[147],"multiple":[148],"Knowledge":[149],"Base":[150],"configurations":[151],"assess":[153],"their":[154],"effectiveness":[155],"supporting":[157],"PT":[158,212],"workflows.":[159],"The":[160],"results":[162,260],"reveal":[163],"notable":[164],"performance":[165],"differences":[166],"among":[167],"evaluated":[169,220],"models.":[170],"Llama":[171,173],"3.1,":[172],"3.2,":[174],"Mistral,":[175],"Falcon":[177,239],"3":[178,193,240],"consistently":[179],"produced":[180],"well-structured,":[181],"step-by-step":[182],"responses":[183,277],"strong":[185],"depth":[187],"command-level":[189],"while":[191,248],"Gemma":[192],"Command":[195],"R":[196],"generated":[197],"coherent":[198],"but":[199],"less":[200],"detailed":[201],"outputs.":[202],"DeepSeek,":[203],"despite":[204],"fewer":[205],"safety":[206],"restrictions,":[207],"failed":[208],"produce":[210,274],"actionable":[211],"guidance.":[213],"To":[214],"further":[215],"validate":[216],"these":[217],"findings,":[218],"models":[221,250],"were":[222],"assessed":[223],"against":[224],"three":[225],"established":[226],"benchmarks":[228],"(AttackSeqBench,":[229],"CyberMetric,":[230],"Cybench)":[232],"confirming":[233],"Mistral":[235],"Small":[236],"3.1":[237],"achieve":[241],"highest":[243],"accuracy":[244],"knowledge-based":[246],"tasks,":[247],"all":[249],"struggle":[251],"interactive,":[253],"flag-":[254],"capture":[255],"(CTF)":[256],"scenarios.":[257,282],"Overall,":[258],"demonstrate":[261],"combining":[263],"carefully":[264],"engineered":[265],"prompts":[266],"RAG-augmented":[268],"retrieval":[269],"enables":[270],"focused,":[275],"domain-relevant":[276],"diverse":[279]},"counts_by_year":[],"updated_date":"2026-06-19T15:47:20.252518","created_date":"2026-05-28T00:00:00"}