{"id":"https://openalex.org/W7225415437","doi":"https://doi.org/10.21203/rs.3.rs-11630562/v1","title":"One Successful Run Is Not an Exploit: Replaying LLM-Written Exploit Scripts with the Model Out of the Loop","display_name":"One Successful Run Is Not an Exploit: Replaying LLM-Written Exploit Scripts with the Model Out of the Loop","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225415437","doi":"https://doi.org/10.21203/rs.3.rs-11630562/v1"},"language":"en","primary_location":{"id":"doi:10.21203/rs.3.rs-11630562/v1","doi":"https://doi.org/10.21203/rs.3.rs-11630562/v1","is_oa":true,"landing_page_url":"https://doi.org/10.21203/rs.3.rs-11630562/v1","pdf_url":"https://www.researchsquare.com/article/rs-11630562/latest.pdf","source":{"id":"https://openalex.org/S4306525896","display_name":"Research Square","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Research Square","raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.researchsquare.com/article/rs-11630562/latest.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102117755","display_name":"Wensheng Huang","orcid":"https://orcid.org/0009-0004-5845-6665","observed_orcids":["https://orcid.org/0009-0004-5845-6665"]},"institutions":[{"id":"https://openalex.org/I4405281061","display_name":"Nanjing City Vocational College","ror":"https://ror.org/03tbvd925","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405281061"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Wensheng Huang","raw_affiliation_strings":["Nanjing City Vocational College"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing City Vocational College","institution_ids":["https://openalex.org/I4405281061"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5102117755"],"corresponding_institution_ids":["https://openalex.org/I4405281061"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.47189998626708984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.47189998626708984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.4359999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.026000000536441803,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.8009999990463257},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.5350000262260437}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7562000155448914},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.7538999915122986},{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.5985999703407288},{"id":"https://openalex.org/C157486923","wikidata":"https://www.wikidata.org/wiki/Q1376436","display_name":"String (physics)","level":2,"score":0.5893999934196472},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.5759999752044678},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.46140000224113464},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.42669999599456787},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.4244999885559082},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.4207000136375427},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4146000146865845},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.37700000405311584},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.35670000314712524},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.3546000123023987},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.34369999170303345},{"id":"https://openalex.org/C2776730729","wikidata":"https://www.wikidata.org/wiki/Q1392130","display_name":"Flag (linear algebra)","level":3,"score":0.3384999930858612},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3124000132083893},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.3089999854564667},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.29010000824928284},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2615000009536743},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2581000030040741}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.21203/rs.3.rs-11630562/v1","doi":"https://doi.org/10.21203/rs.3.rs-11630562/v1","is_oa":true,"landing_page_url":"https://doi.org/10.21203/rs.3.rs-11630562/v1","pdf_url":"https://www.researchsquare.com/article/rs-11630562/latest.pdf","source":{"id":"https://openalex.org/S4306525896","display_name":"Research Square","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Research Square","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.21203/rs.3.rs-11630562/v1","doi":"https://doi.org/10.21203/rs.3.rs-11630562/v1","is_oa":true,"landing_page_url":"https://doi.org/10.21203/rs.3.rs-11630562/v1","pdf_url":"https://www.researchsquare.com/article/rs-11630562/latest.pdf","source":{"id":"https://openalex.org/S4306525896","display_name":"Research Square","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Research Square","raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7225415437.pdf","grobid_xml":"https://content.openalex.org/works/W7225415437.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W2026926213","https://openalex.org/W3122178585","https://openalex.org/W3177813494","https://openalex.org/W4399836817","https://openalex.org/W4403118443","https://openalex.org/W4408980996","https://openalex.org/W4414428062","https://openalex.org/W4415314082","https://openalex.org/W4415653990","https://openalex.org/W4415967406","https://openalex.org/W7092291230","https://openalex.org/W7133184436","https://openalex.org/W7140102743","https://openalex.org/W7152533497","https://openalex.org/W7171400955","https://openalex.org/W7171743252","https://openalex.org/W7171819450","https://openalex.org/W7172499856","https://openalex.org/W7204526542","https://openalex.org/W7214096218","https://openalex.org/W7214753479","https://openalex.org/W7215133701","https://openalex.org/W7220371477","https://openalex.org/W7221198112","https://openalex.org/W7222293283"],"related_works":["https://openalex.org/W7206193539","https://openalex.org/W7172208541","https://openalex.org/W7213590683","https://openalex.org/W7165473354","https://openalex.org/W7223993361","https://openalex.org/W7162776720","https://openalex.org/W7167415300","https://openalex.org/W7165474193","https://openalex.org/W7222670283","https://openalex.org/W7222893041"],"abstract_inverted_index":{"Abstract":[0],"Offensive-security":[1],"benchmarks":[2],"for":[3],"large":[4],"language":[5],"model":[6,41,61],"(LLM)":[7],"agents":[8],"score":[9],"a":[10,15,88,153,158],"run":[11,32],"as":[12,115],"successful":[13],"when":[14,143],"flag":[16,97],"string":[17],"appears":[18],"in":[19,98,148,157,176],"its":[20,144],"output;":[21],"published":[22],"systems":[23],"commonly":[24],"stop":[25],"at":[26,37,121,184],"the":[27,40,49,56,60,64,96,133,180],"first":[28],"success.":[29],"One":[30],"verified":[31],"leaves":[33],"two":[34,57],"questions":[35],"open":[36],"once:":[38],"whether":[39,48],"will":[42],"produce":[43],"that":[44],"result":[45],"again,":[46],"and":[47,77,140,194,201],"script":[50,79],"itself":[51],"still":[52],"works.":[53],"We":[54],"separate":[55],"by":[58],"taking":[59],"out":[62],"of":[63,100,179],"loop.":[65],"Across":[66],"14":[67],"web":[68],"challenges,":[69],"280":[70],"agent":[71,197],"runs":[72],"yielded":[73],"196":[74],"winning":[75],"scripts,":[76],"each":[78],"was":[80],"replayed":[81,147],"verbatim,":[82],"R":[83],"=":[84,123,171],"10":[85],"times,":[86],"into":[87],"freshly":[89],"started":[90],"target":[91,160],"container.":[92],"Cold":[93],"replay":[94],"recovered":[95],"1,950":[99],"1,960":[101],"replays":[102],"(99.49\\%,":[103],"task-clustered":[104],"bootstrap":[105],"95\\%":[106],"confidence":[107],"interval":[108],"98.37":[109],"to":[110],"100.00),":[111],"classifying":[112],"195":[113],"artifacts":[114],"stable,":[116],"none":[117],"flaky,":[118],"one":[119,162,186],"dead":[120,130],"(R":[122],"10,":[124],"cold,":[125],"reference":[126],"environment).":[127],"The":[128],"single":[129],"artifact":[131,193],"is":[132,146],"informative":[134],"case:":[135],"it":[136,151,166],"fails":[137],"0/10":[138],"alone":[139],"succeeds":[141],"10/10":[142],"trajectory":[145],"order,":[149],"because":[150],"hard-codes":[152],"subscript":[154],"valid":[155],"only":[156],"primed":[159],"process;":[161],"extra":[163],"request":[164],"flips":[165],"(Fisher":[167],"exact":[168],"test,":[169],"p":[170],"1.08e-05).":[172],"Instability":[173],"instead":[174],"concentrates":[175],"generation:":[177],"12":[178],"24":[181],"cells":[182],"with":[183],"least":[185],"hit":[187],"are":[188],"flaky":[189],"there.":[190],"Replaying":[191],"an":[192,196],"re-running":[195],"measure":[198],"different":[199],"quantities,":[200],"single-run":[202],"scoring":[203],"reports":[204],"neither.":[205]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-11-07T00:00:00"}