{"id":"https://openalex.org/W7162507072","doi":"https://doi.org/10.48550/arxiv.2605.26548","title":"SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks?","display_name":"SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks?","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162507072","doi":"https://doi.org/10.48550/arxiv.2605.26548"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.26548","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.26548","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.26548","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137121943","display_name":"Hwiwon Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Hwiwon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137163264","display_name":"Jiawei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137100809","display_name":"Dongjun Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Dongjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112854551","display_name":"Wubing Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Wubing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141746201","display_name":"Ziqi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ziqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141334459","display_name":"Chunqiu Steven Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Chunqiu Steven","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5141445227","display_name":"Lingming Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Lingming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.3093000054359436,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.3093000054359436,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.1882999986410141,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.12060000002384186,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-security-assurance","display_name":"Software security assurance","score":0.6696000099182129},{"id":"https://openalex.org/keywords/fuzz-testing","display_name":"Fuzz testing","score":0.6661999821662903},{"id":"https://openalex.org/keywords/security-bug","display_name":"Security bug","score":0.6485999822616577},{"id":"https://openalex.org/keywords/vulnerability","display_name":"Vulnerability (computing)","score":0.5503000020980835},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5289999842643738},{"id":"https://openalex.org/keywords/secure-coding","display_name":"Secure coding","score":0.4781000018119812},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.47119998931884766},{"id":"https://openalex.org/keywords/vulnerability-assessment","display_name":"Vulnerability assessment","score":0.43290001153945923}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7459999918937683},{"id":"https://openalex.org/C62913178","wikidata":"https://www.wikidata.org/wiki/Q7554361","display_name":"Software security assurance","level":4,"score":0.6696000099182129},{"id":"https://openalex.org/C111065885","wikidata":"https://www.wikidata.org/wiki/Q1189053","display_name":"Fuzz testing","level":3,"score":0.6661999821662903},{"id":"https://openalex.org/C131275738","wikidata":"https://www.wikidata.org/wiki/Q7445023","display_name":"Security bug","level":5,"score":0.6485999822616577},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.5503000020980835},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5289999842643738},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.5026000142097473},{"id":"https://openalex.org/C22680326","wikidata":"https://www.wikidata.org/wiki/Q7444867","display_name":"Secure coding","level":5,"score":0.4781000018119812},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.47119998931884766},{"id":"https://openalex.org/C167063184","wikidata":"https://www.wikidata.org/wiki/Q1400839","display_name":"Vulnerability assessment","level":3,"score":0.43290001153945923},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4320000112056732},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4300000071525574},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.4138999879360199},{"id":"https://openalex.org/C172776598","wikidata":"https://www.wikidata.org/wiki/Q7943570","display_name":"Vulnerability management","level":4,"score":0.4011000096797943},{"id":"https://openalex.org/C121822524","wikidata":"https://www.wikidata.org/wiki/Q5157582","display_name":"Computer security model","level":2,"score":0.3662000000476837},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.3499999940395355},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.30300000309944153},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.28279998898506165},{"id":"https://openalex.org/C140547941","wikidata":"https://www.wikidata.org/wiki/Q7797194","display_name":"Threat model","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.262800008058548}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.26548","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.26548","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.26548","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.26548","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Finding":[0],"a":[1,8,21,36,59,83,194,202],"real":[2,45],"vulnerability":[3,114,234],"in":[4,189],"complicated":[5],"systems":[6],"is":[7,259],"challenging,":[9],"long-horizon":[10,221,233],"task":[11,57],"that":[12,38,148,197],"demands":[13],"reasoning":[14],"across":[15,99],"an":[16],"entire":[17],"codebase":[18],"to":[19,155,218],"produce":[20],"working":[22,49],"proof-of-concept":[23],"(PoC).":[24],"However,":[25],"such":[26],"critical":[27,113],"security":[28,222],"problems":[29],"remain":[30],"understudied.":[31],"We":[32,68,91,145],"present":[33],"SEC-bench":[34,93,183,211,229],"Pro,":[35],"benchmark":[37],"measures":[39],"how":[40,241],"well":[41],"frontier":[42,126],"models":[43,130,167],"hunt":[44],"vulnerabilities":[46,98,188],"by":[47,216],"reproducing":[48],"PoC":[50],"inputs":[51],"from":[52],"disclosed":[53],"reports,":[54],"where":[55,232,237],"each":[56],"pairs":[58],"concrete":[60],"bug":[61],"with":[62,95,138,151],"the":[63,71,102,109,135,176,220,246],"instructions":[64],"for":[65,77,87,169,251],"triggering":[66],"it.":[67],"also":[69,146,185],"demonstrate":[70],"limitations":[72],"of":[73,142,175,224],"existing":[74],"rule-based":[75],"judges":[76],"grading":[78,243],"generated":[79],"PoCs,":[80],"and":[81,104,108,121,128,131,181,191,200,240,255],"propose":[82],"novel":[84],"LLM-based":[85],"judge":[86],"more":[88],"precise":[89],"grading.":[90],"instantiate":[92],"Pro":[94,184,212,230],"344":[96,177],"validated":[97],"three":[100,132,187],"targets,":[101],"V8":[103,190],"SpiderMonkey":[105],"browser":[106],"engines":[107],"Linux":[110],"kernel,":[111],"covering":[112],"families":[115],"including":[116,193],"memory-safety,":[117],"sandbox,":[118],"JIT,":[119],"race-condition,":[120],"kernel-subsystem":[122],"bugs.":[123],"Across":[124],"six":[125],"commercial":[127],"open-weight":[129,166],"coding":[133],"agents,":[134],"strongest,":[136],"Codex":[137],"GPT-5.5,":[139],"solves":[140,159,172],"58%":[141],"instances":[143,161],"overall.":[144],"observe":[147],"Claude":[149],"Code":[150],"Opus":[152],"4.6":[153],"tends":[154],"time":[156],"out":[157],"but":[158],"most":[160],"it":[162,238],"completes.":[163],"In":[164],"contrast,":[165],"struggle;":[168],"example,":[170],"GLM-5":[171],"only":[173],"13":[174],"instances.":[178],"During":[179],"construction":[180],"evaluation,":[182],"surfaced":[186],"SpiderMonkey,":[192],"sandbox":[195],"escape":[196],"was":[198],"fixed":[199],"earned":[201],"$20,000":[203],"Google":[204],"Vulnerability":[205],"Reward":[206],"Program":[207],"bounty.":[208],"More":[209],"recently,":[210],"has":[213],"been":[214],"adopted":[215],"OpenAI":[217],"evaluate":[219],"capabilities":[223],"its":[225],"newest":[226],"models.":[227],"Overall,":[228],"exposes":[231],"discovery":[235],"succeeds,":[236],"fails,":[239],"different":[242],"choices":[244],"change":[245],"evaluation":[247,254],"landscape,":[248],"offering":[249],"insights":[250],"security-centric":[252],"model":[253],"training.":[256],"Our":[257],"artifact":[258],"available":[260],"at":[261],"https://github.com/SEC-bench/SEC-bench-Pro.":[262]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-28T00:00:00"}