{"id":"https://openalex.org/W4396790452","doi":"https://doi.org/10.1109/satml59370.2024.00040","title":"CodeLMSec Benchmark: Systematically Evaluating and Finding Security Vulnerabilities in Black-Box Code Language Models","display_name":"CodeLMSec Benchmark: Systematically Evaluating and Finding Security Vulnerabilities in Black-Box Code Language Models","alternate_titles":[],"publication_year":2024,"publication_date":"2024-04-09","ids":{"openalex":"https://openalex.org/W4396790452","doi":"https://doi.org/10.1109/satml59370.2024.00040"},"language":"en","primary_location":{"id":"doi:10.1109/satml59370.2024.00040","doi":"https://doi.org/10.1109/satml59370.2024.00040","is_oa":false,"landing_page_url":"https://doi.org/10.1109/satml59370.2024.00040","pdf_url":null,"source":{"id":"https://openalex.org/S7407087861","display_name":"IEEE Conference on Secure and Trustworthy Machine Learning (SaTML)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4396790452"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2302.04012","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075967174","display_name":"Hossein Hajipour","orcid":"https://orcid.org/0000-0001-6911-9390","observed_orcids":["https://orcid.org/0000-0001-6911-9390"]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Hossein Hajipour","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056634204","display_name":"Keno Ha\u00dfler","orcid":"https://orcid.org/0009-0008-9115-2769","observed_orcids":["https://orcid.org/0009-0008-9115-2769"]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Keno Hassler","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056790702","display_name":"Thorsten Holz","orcid":"https://orcid.org/0000-0002-2783-1264","observed_orcids":["https://orcid.org/0000-0002-2783-1264"]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Thorsten Holz","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049219646","display_name":"Lea Sch\u00f6nherr","orcid":"https://orcid.org/0000-0003-3779-7781","observed_orcids":["https://orcid.org/0000-0003-3779-7781"]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lea Sch\u00f6nherr","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003887059","display_name":"Mario Fritz","orcid":"https://orcid.org/0000-0001-8949-9896","observed_orcids":["https://orcid.org/0000-0001-8949-9896"]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Mario Fritz","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security","institution_ids":["https://openalex.org/I4210128801"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210128801"],"apc_list":null,"apc_paid":null,"fwci":19.1532,"has_fulltext":true,"cited_by_count":133,"citation_normalized_percentile":{"value":0.99570945,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"684","last_page":"709"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5268999934196472,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5268999934196472,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.17470000684261322,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.11900000274181366,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-language-models","display_name":"code language models","score":0.8180000185966492},{"id":"https://openalex.org/keywords/security-vulnerabilities","display_name":"security vulnerabilities","score":0.824999988079071},{"id":"https://openalex.org/keywords/black-box-evaluation","display_name":"black-box evaluation","score":0.6949999928474426},{"id":"https://openalex.org/keywords/few-shot-prompting","display_name":"few-shot prompting","score":0.9399999976158142},{"id":"https://openalex.org/keywords/benchmark-dataset","display_name":"benchmark dataset","score":0.6010000109672546},{"id":"https://openalex.org/keywords/training-data-contamination","display_name":"training data contamination","score":0.6959999799728394},{"id":"https://openalex.org/keywords/ai-assisted-programming","display_name":"AI-assisted programming","score":0.7570000290870667},{"id":"https://openalex.org/keywords/github-copilot","display_name":"GitHub Copilot","score":0.6620000004768372},{"id":"https://openalex.org/keywords/code-generation-security","display_name":"code generation security","score":0.414000004529953}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8682726621627808},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5722898244857788},{"id":"https://openalex.org/C22680326","wikidata":"https://www.wikidata.org/wiki/Q7444867","display_name":"Secure coding","level":5,"score":0.49015793204307556},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.4784705340862274},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.46892625093460083},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.457028329372406},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4569575786590576},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.4214002192020416},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4127384126186371},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.39714083075523376},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.35373884439468384},{"id":"https://openalex.org/C62913178","wikidata":"https://www.wikidata.org/wiki/Q7554361","display_name":"Software security assurance","level":4,"score":0.3500741124153137},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29508668184280396},{"id":"https://openalex.org/C527648132","wikidata":"https://www.wikidata.org/wiki/Q189900","display_name":"Information security","level":2,"score":0.17807120084762573},{"id":"https://openalex.org/C29983905","wikidata":"https://www.wikidata.org/wiki/Q7445066","display_name":"Security service","level":3,"score":0.08074256777763367},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.0}],"mesh":[],"locations_count":8,"locations":[{"id":"doi:10.1109/satml59370.2024.00040","doi":"https://doi.org/10.1109/satml59370.2024.00040","is_oa":false,"landing_page_url":"https://doi.org/10.1109/satml59370.2024.00040","pdf_url":null,"source":{"id":"https://openalex.org/S7407087861","display_name":"IEEE Conference on Secure and Trustworthy Machine Learning (SaTML)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2302.04012","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.04012","pdf_url":"https://arxiv.org/pdf/2302.04012","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:figshare.com:article/25532854","doi":null,"is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/CodeLMSec_Benchmark_Systematically_Evaluating_and_Finding_Security_Vulnerabilities_in_Black-Box_Code_Language_Models/25532854","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference contribution"},{"id":"pmh:oai:figshare.com:article/32772348","doi":null,"is_oa":true,"landing_page_url":"https://figshare.com/articles/preprint/CodeLMSec_Benchmark_Systematically_Evaluating_and_Finding_Security_Vulnerabilities_in_Black-Box_Code_Language_Models/32772348","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},{"id":"doi:10.48550/arxiv.2302.04012","doi":"https://doi.org/10.48550/arxiv.2302.04012","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2302.04012","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.60882/cispa.25532854.v1","doi":"https://doi.org/10.60882/cispa.25532854.v1","is_oa":true,"landing_page_url":"https://doi.org/10.60882/cispa.25532854.v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050916","display_name":"CISPA Helmholtz Center","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"ConferenceProceeding"},{"id":"doi:10.60882/cispa.32772348","doi":"https://doi.org/10.60882/cispa.32772348","is_oa":true,"landing_page_url":"https://doi.org/10.60882/cispa.32772348","pdf_url":null,"source":{"id":"https://openalex.org/S7407050916","display_name":"CISPA Helmholtz Center","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.60882/cispa.32772348.v1","doi":"https://doi.org/10.60882/cispa.32772348.v1","is_oa":true,"landing_page_url":"https://doi.org/10.60882/cispa.32772348.v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050916","display_name":"CISPA Helmholtz Center","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2302.04012","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.04012","pdf_url":"https://arxiv.org/pdf/2302.04012","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3562914802","display_name":"European Lighthouse on Secure and Safe AI","funder_award_id":"101070617","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321408","display_name":"Ministry of Education","ror":"https://ror.org/01p262204"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4396790452.pdf","grobid_xml":"https://content.openalex.org/works/W4396790452.grobid-xml"},"referenced_works_count":60,"referenced_works":["https://openalex.org/W814172419","https://openalex.org/W1761184020","https://openalex.org/W1852477053","https://openalex.org/W1989657183","https://openalex.org/W2051267297","https://openalex.org/W2117798902","https://openalex.org/W2141736632","https://openalex.org/W2156279557","https://openalex.org/W2158297335","https://openalex.org/W2350778671","https://openalex.org/W2514084604","https://openalex.org/W2514974017","https://openalex.org/W2896457183","https://openalex.org/W2938704169","https://openalex.org/W2963341956","https://openalex.org/W2963594498","https://openalex.org/W2972552958","https://openalex.org/W2981852735","https://openalex.org/W2982553922","https://openalex.org/W3030163527","https://openalex.org/W3034957837","https://openalex.org/W3048197573","https://openalex.org/W3086007799","https://openalex.org/W3091588759","https://openalex.org/W3098605233","https://openalex.org/W3119507053","https://openalex.org/W3156470785","https://openalex.org/W3170092793","https://openalex.org/W3177813494","https://openalex.org/W3198685994","https://openalex.org/W4224060952","https://openalex.org/W4226278401","https://openalex.org/W4226287673","https://openalex.org/W4226485558","https://openalex.org/W4252326246","https://openalex.org/W4281611258","https://openalex.org/W4281763794","https://openalex.org/W4282830668","https://openalex.org/W4285490477","https://openalex.org/W4288057765","https://openalex.org/W4292956935","https://openalex.org/W4297900086","https://openalex.org/W4304731983","https://openalex.org/W4307001524","https://openalex.org/W4307079201","https://openalex.org/W4308627645","https://openalex.org/W4315815628","https://openalex.org/W4320560161","https://openalex.org/W4327810158","https://openalex.org/W4367000491","https://openalex.org/W4376167329","https://openalex.org/W4380993527","https://openalex.org/W4384918448","https://openalex.org/W4385187279","https://openalex.org/W4385570182","https://openalex.org/W4386185625","https://openalex.org/W4386982649","https://openalex.org/W6798182279","https://openalex.org/W7133198585","https://openalex.org/W7133211372"],"related_works":["https://openalex.org/W7165655095","https://openalex.org/W4417097196","https://openalex.org/W4401990041","https://openalex.org/W7124946063","https://openalex.org/W4415200955","https://openalex.org/W4387963719","https://openalex.org/W4384026520","https://openalex.org/W4403573323","https://openalex.org/W7130651357","https://openalex.org/W4403646794"],"abstract_inverted_index":{"Large":[0],"language":[1,83,142,191,230],"models":[2,56,84,99,143,192],"(LLMs)":[3],"for":[4,54,104,211],"automatic":[5],"code":[6,94,141,164,170,190,229],"generation":[7,95,171],"have":[8,22,38,100],"recently":[9],"achieved":[10],"breakthroughs":[11],"in":[12,18,168,193],"several":[13],"programming":[14,20,45],"tasks.":[15],"Their":[16],"advances":[17],"competition-level":[19],"problems":[21],"made":[23],"them":[24,91],"an":[25],"essential":[26],"pillar":[27],"of":[28,42,50,116,125,140,185,207,228],"AI-assisted":[29],"pair":[30],"programming,":[31],"and":[32,67,73,89,119,223],"tools":[33],"such":[34],"as":[35,40,218],"GitHub":[36],"Copilot":[37],"emerged":[39],"part":[41],"the":[43,61,82,93,122,137,157,183,225],"daily":[44],"workflow":[46],"used":[47],"by":[48,188],"millions":[49],"developers.":[51],"Training":[52],"data":[53,79],"these":[55,87,98,126],"is":[57,68],"usually":[58],"collected":[59],"from":[60,64],"Internet":[62],"(e.g.,":[63],"open-source":[65],"repositories)":[66],"likely":[69],"to":[70,85,107,134,144,148,160,203,221],"contain":[71],"faults":[72],"security":[74,123,138,196,226],"vulnerabilities.":[75],"This":[76,173,215],"unsanitized":[77],"training":[78],"can":[80],"cause":[81],"learn":[86],"vulnerabilities":[88,167],"propagate":[90],"during":[92],"procedure.":[96],"While":[97],"been":[101],"extensively":[102],"evaluated":[103],"their":[105,146],"ability":[106],"produce":[108],"functionally":[109],"correct":[110],"programs,":[111],"there":[112],"remains":[113],"a":[114,132,176,205,219],"lack":[115],"comprehensive":[117],"investigations":[118],"benchmarks":[120],"addressing":[121],"aspects":[124],"models.In":[127],"this":[128,153],"work,":[129],"we":[130,155,199],"propose":[131],"method":[133,202],"systematically":[135],"study":[136],"issues":[139],"assess":[145],"susceptibility":[147],"generating":[149,194],"vulnerable":[150],"code.":[151],"To":[152],"end,":[154],"introduce":[156],"first":[158],"approach":[159,187],"automatically":[161],"find":[162],"generated":[163],"that":[165],"contains":[166],"black-box":[169],"models.":[172,231],"involves":[174],"proposing":[175],"novel":[177],"few-shot":[178],"prompting":[179],"approach.":[180],"We":[181],"evaluate":[182,222],"effectiveness":[184],"our":[186,201],"examining":[189],"high-risk":[195],"weaknesses.":[197],"Furthermore,":[198],"use":[200],"create":[204],"collection":[206],"diverse":[208],"non-secure":[209],"prompts":[210],"various":[212],"vulnerability":[213],"scenarios.":[214],"dataset":[216],"serves":[217],"benchmark":[220],"compare":[224],"weaknesses":[227]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4396790452","counts_by_year":[{"year":2026,"cited_by_count":69},{"year":2025,"cited_by_count":33},{"year":2024,"cited_by_count":27},{"year":2023,"cited_by_count":4}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2024-05-11T00:00:00"}
