{"id":"https://openalex.org/W4410537167","doi":"https://doi.org/10.1109/icst62969.2025.10988968","title":"Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities","display_name":"Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities","publication_year":2025,"publication_date":"2025-03-31","ids":{"openalex":"https://openalex.org/W4410537167","doi":"https://doi.org/10.1109/icst62969.2025.10988968"},"language":"en","primary_location":{"id":"doi:10.1109/icst62969.2025.10988968","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst62969.2025.10988968","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058476758","display_name":"Avishree Khare","orcid":"https://orcid.org/0009-0006-9217-3559"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Avishree Khare","raw_affiliation_strings":["University of Pennsylvania,Philadelphia,USA"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063258857","display_name":"Saikat Dutta","orcid":"https://orcid.org/0000-0001-6021-5407"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saikat Dutta","raw_affiliation_strings":["Cornell University,Ithaca,USA"],"affiliations":[{"raw_affiliation_string":"Cornell University,Ithaca,USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100422866","display_name":"Ziyang Li","orcid":"https://orcid.org/0009-0001-3084-1308"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ziyang Li","raw_affiliation_strings":["University of Pennsylvania,Philadelphia,USA"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063952286","display_name":"Alaia Solko-Breslin","orcid":null},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alaia Solko-Breslin","raw_affiliation_strings":["University of Pennsylvania,Philadelphia,USA"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065059795","display_name":"Rajeev Alur","orcid":"https://orcid.org/0000-0003-1733-7083"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rajeev Alur","raw_affiliation_strings":["University of Pennsylvania,Philadelphia,USA"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075879790","display_name":"Mayur Naik","orcid":"https://orcid.org/0000-0003-1348-8618"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mayur Naik","raw_affiliation_strings":["University of Pennsylvania,Philadelphia,USA"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania,Philadelphia,USA","institution_ids":["https://openalex.org/I79576946"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5058476758"],"corresponding_institution_ids":["https://openalex.org/I79576946"],"apc_list":null,"apc_paid":null,"fwci":80.4197,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.99935805,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"103","last_page":"114"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.7915999889373779,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.7915999889373779,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.7785999774932861,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7195000052452087,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7366898059844971},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.43164852261543274},{"id":"https://openalex.org/keywords/secure-coding","display_name":"Secure coding","score":0.4173860549926758},{"id":"https://openalex.org/keywords/software-security-assurance","display_name":"Software security assurance","score":0.27895087003707886},{"id":"https://openalex.org/keywords/information-security","display_name":"Information security","score":0.2048695683479309},{"id":"https://openalex.org/keywords/security-service","display_name":"Security service","score":0.09825751185417175}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7366898059844971},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.43164852261543274},{"id":"https://openalex.org/C22680326","wikidata":"https://www.wikidata.org/wiki/Q7444867","display_name":"Secure coding","level":5,"score":0.4173860549926758},{"id":"https://openalex.org/C62913178","wikidata":"https://www.wikidata.org/wiki/Q7554361","display_name":"Software security assurance","level":4,"score":0.27895087003707886},{"id":"https://openalex.org/C527648132","wikidata":"https://www.wikidata.org/wiki/Q189900","display_name":"Information security","level":2,"score":0.2048695683479309},{"id":"https://openalex.org/C29983905","wikidata":"https://www.wikidata.org/wiki/Q7445066","display_name":"Security service","level":3,"score":0.09825751185417175}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icst62969.2025.10988968","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst62969.2025.10988968","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.5400000214576721,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W2025411198","https://openalex.org/W2885030880","https://openalex.org/W2998879504","https://openalex.org/W3091588759","https://openalex.org/W3166095789","https://openalex.org/W3177116043","https://openalex.org/W3183469243","https://openalex.org/W4240399292","https://openalex.org/W4252326246","https://openalex.org/W4285490489","https://openalex.org/W4308643319","https://openalex.org/W4311165836","https://openalex.org/W4312436517","https://openalex.org/W4312969325","https://openalex.org/W4378591002","https://openalex.org/W4382239980","https://openalex.org/W4384302789","https://openalex.org/W4384304865","https://openalex.org/W4384345708","https://openalex.org/W4387298393","https://openalex.org/W4389159862","https://openalex.org/W4391558363","https://openalex.org/W4391579639","https://openalex.org/W4396242417","https://openalex.org/W4398785927","https://openalex.org/W4402264467","https://openalex.org/W6729167014","https://openalex.org/W6767260250","https://openalex.org/W6809646742","https://openalex.org/W6838461927","https://openalex.org/W6853209900"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W1631032283","https://openalex.org/W1643546019","https://openalex.org/W2167151567"],"abstract_inverted_index":{"Security":[0],"vulnerabilities":[1,164,187],"in":[2,116,139,214],"modern":[3],"software":[4],"are":[5,159],"prevalent":[6],"and":[7,20,35,68,75,77,84,113,118,120,134,150,176,247,249],"harmful.":[8],"While":[9],"automated":[10],"vulnerability":[11,85,124,275],"detection":[12,80,276],"techniques":[13],"have":[14],"made":[15],"promising":[16,231],"progress,":[17],"their":[18],"scalability":[19],"applicability":[21],"remain":[22],"challenging.":[23],"The":[24],"remarkable":[25],"performance":[26,81,208],"of":[27,72,92,148,153,209,216,236],"Large":[28],"Language":[29],"Models":[30],"(LLMs),":[31],"such":[32,171,193,240],"as":[33,172,194,241],"GPT-4":[34],"CodeLlama,":[36],"on":[37,96,185,211,223,273],"code-related":[38],"tasks":[39],"has":[40],"prompted":[41],"recent":[42],"works":[43],"to":[44,51,221,254,259],"explore":[45],"if":[46,261],"LLMs":[47,95,130,158,181,210,229],"can":[48,269],"be":[49],"used":[50],"detect":[52],"security":[53,106],"vulnerabilities.":[54],"In":[55],"this":[56],"paper,":[57],"we":[58,88,226],"perform":[59],"a":[60,66],"more":[61,69],"comprehensive":[62],"study":[63],"by":[64],"examining":[65],"larger":[67],"diverse":[70,105],"set":[71],"datasets,":[73],"languages,":[74],"LLMs,":[76],"qualitatively":[78],"evaluating":[79],"across":[82,131,155],"prompts":[83],"classes.":[86,125],"Concretely,":[87],"evaluate":[89],"the":[90,237],"effectiveness":[91,138],"16":[93],"pre-trained":[94],"5,000":[97],"code":[98,256,262],"samples-1,000":[99],"randomly":[100],"selected":[101],"each":[102],"from":[103],"five":[104],"datasets.":[107,157],"These":[108],"balanced":[109],"datasets":[110,213],"encompass":[111],"synthetic":[112],"real-world":[114,212],"projects":[115],"Java":[117],"C/C++":[119],"cover":[121],"25":[122],"distinct":[123],"Our":[126],"results":[127],"show":[128,136,230],"that":[129,165,198,202,228],"all":[132,156],"scales":[133],"families":[135],"modest":[137],"end-to-end":[140],"reasoning":[141],"about":[142],"vul-nerabilities,":[143],"obtaining":[144],"an":[145],"average":[146],"accuracy":[147],"62.8%":[149],"F1":[151,217],"score":[152,218],"0.71":[154],"significantly":[160,206],"better":[161],"at":[162,233],"detecting":[163],"typically":[166],"only":[167],"need":[168],"intra-procedural":[169],"reasoning,":[170],"OS":[173],"Command":[174],"Injection":[175],"NULL":[177],"Pointer":[178],"Dereference.":[179],"Moreover,":[180],"report":[182],"higher":[183],"accuracies":[184],"these":[186],"than":[188],"popular":[189],"static":[190],"analysis":[191,205,238],"tools,":[192],"CodeQL.":[195],"We":[196,265],"find":[197],"advanced":[199],"prompting":[200],"strategies":[201],"involve":[203],"step-by-step":[204],"improve":[207],"terms":[215],"(by":[219],"up":[220],"0.18":[222],"average).":[224],"Interestingly,":[225],"observe":[227],"abilities":[232],"performing":[234],"parts":[235],"correctly,":[239],"identifying":[242],"vulnerability-related":[243],"specifications":[244],"(e.g.,":[245,258],"sources":[246],"sinks)":[248],"leveraging":[250],"natural":[251],"language":[252],"information":[253],"understand":[255],"behavior":[257],"check":[260],"is":[263],"sanitized).":[264],"believe":[266],"our":[267],"insights":[268],"motivate":[270],"future":[271],"work":[272],"LLM-augmented":[274],"systems.":[277]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":23}],"updated_date":"2026-04-01T17:29:45.350535","created_date":"2025-10-10T00:00:00"}