{"id":"https://openalex.org/W7166819775","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1786","title":"RealSec-bench: A Benchmark for Evaluating Secure Code Generation in Real-World Repositories","display_name":"RealSec-bench: A Benchmark for Evaluating Secure Code Generation in Real-World Repositories","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166819775","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1786"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.1786","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1786","pdf_url":"https://aclanthology.org/2026.findings-acl.1786.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-acl.1786.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139787498","display_name":"Yanlin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yanlin Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139783817","display_name":"Ziyao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ziyao Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139786103","display_name":"Chong Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chong Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139776514","display_name":"Xinyi Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xinyi Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139827205","display_name":"Mingwei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mingwei Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139833813","display_name":"Yong Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yong Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139766087","display_name":"Jiachi Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiachi Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139739741","display_name":"Zibin Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zibin Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.80942122,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"35866","last_page":"35883"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.17550000548362732,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.17550000548362732,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.1476999968290329,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.14339999854564667,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5146999955177307},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.46209999918937683},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.31369999051094055},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.3082999885082245},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.30160000920295715}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6869999766349792},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5146999955177307},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.46209999918937683},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.35179999470710754},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.31369999051094055},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.31119999289512634},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.3082999885082245},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.30160000920295715},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2906999886035919},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.25459998846054077},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.25450000166893005},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2526000142097473},{"id":"https://openalex.org/C47487241","wikidata":"https://www.wikidata.org/wiki/Q5227230","display_name":"Data access","level":2,"score":0.2526000142097473},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.1786","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1786","pdf_url":"https://aclanthology.org/2026.findings-acl.1786.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.1786","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1786","pdf_url":"https://aclanthology.org/2026.findings-acl.1786.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.42280223965644836,"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals"}],"awards":[{"id":"https://openalex.org/G1292579918","display_name":null,"funder_award_id":"92582202","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4425133490","display_name":null,"funder_award_id":"62302534","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4672339227","display_name":null,"funder_award_id":"92582117","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321160","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166819775.pdf","grobid_xml":"https://content.openalex.org/works/W7166819775.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"capabilities":[7],"in":[8,14,36,49,99,198],"code":[9,17,63,196],"generation,":[10],"but":[11],"their":[12],"proficiency":[13],"producing":[15],"secure":[16,62,195],"remains":[18],"a":[19,58,74,112,139],"critical,":[20],"under-explored":[21],"area.Existing":[22],"benchmarks":[23],"often":[24,176],"fall":[25],"short":[26],"by":[27],"relying":[28],"on":[29,134],"synthetic":[30],"vulnerabilities":[31,120],"or":[32],"evaluating":[33],"functional":[34,147,161,182,193],"correctness":[35,148,183],"isolation,":[37],"failing":[38],"to":[39,123,144,167,178],"capture":[40],"the":[41,190],"complex":[42],"interplay":[43],"between":[44,192],"functionality":[45],"and":[46,88,110,149,194],"security":[47,150,174],"found":[48],"real-world":[50],"software.To":[51],"address":[52],"this":[53],"gap,":[54],"we":[55,128],"introduce":[56,138],"RealSec-bench,":[57,127],"new":[59],"benchmark":[60,94],"for":[61],"generation":[64,197],"meticulously":[65],"constructed":[66],"from":[67],"real-world,":[68],"highrisk":[69],"Java":[70],"repositories.Our":[71],"methodology":[72],"employs":[73],"multi-stage":[75],"pipeline":[76],"that":[77,153],"combines":[78],"systematic":[79],"SAST":[80],"scanning":[81],"with":[82,121,172],"CodeQL,":[83],"LLMbased":[84],"false":[85],"positive":[86],"elimination,":[87],"rigorous":[89],"human":[90],"expert":[91],"validation.The":[92],"resulting":[93],"contains":[95],"105":[96],"instances":[97],"grounded":[98],"realword":[100],"repository":[101],"contexts,":[102],"spanning":[103],"19":[104],"Common":[105],"Weakness":[106],"Enumeration":[107],"(CWE)":[108],"types":[109],"exhibiting":[111],"wide":[113],"diversity":[114],"of":[115],"data":[116],"flow":[117],"complexities,":[118],"including":[119],"up":[122],"34-hop":[124],"inter-procedural":[125],"dependencies.Using":[126],"conduct":[129],"an":[130],"extensive":[131],"empirical":[132],"study":[133],"5":[135],"popular":[136],"LLMs.We":[137],"novel":[140],"composite":[141],"metric,":[142],"SecurePass@k,":[143],"assess":[145],"both":[146],"simultaneously.We":[151],"find":[152],"while":[154],"Retrieval-Augmented":[155],"Generation":[156],"(RAG)":[157],"techniques":[158],"can":[159],"improve":[160],"correctness,":[162],"they":[163],"provide":[164],"negligible":[165],"benefits":[166],"security.Furthermore,":[168],"explicitly":[169],"prompting":[170],"models":[171],"general":[173],"guidelines":[175],"leads":[177],"compilation":[179],"failures,":[180],"harming":[181],"without":[184],"reliably":[185],"preventing":[186],"vulnerabilities.Our":[187],"work":[188],"highlights":[189],"gap":[191],"current":[199],"LLMs.":[200]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}