{"id":"https://openalex.org/W7160898367","doi":"https://doi.org/10.48550/arxiv.2605.09070","title":"Single-Configuration Attack Success Rate Is Not Enough: Jailbreak Evaluations Should Report Distributional Attack Success","display_name":"Single-Configuration Attack Success Rate Is Not Enough: Jailbreak Evaluations Should Report Distributional Attack Success","publication_year":2026,"publication_date":"2026-05-09","ids":{"openalex":"https://openalex.org/W7160898367","doi":"https://doi.org/10.48550/arxiv.2605.09070"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09070","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09070","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09070","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135961655","display_name":"Carsten Maple","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maple, Carsten","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135990456","display_name":"Abhishek Kumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumar, Abhishek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135994698","display_name":"Riya Tapwal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tapwal, Riya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.4821999967098236,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.4821999967098236,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10951","display_name":"Cryptographic Implementations and Security","score":0.14820000529289246,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.08659999817609787,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5394999980926514},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4763999879360199},{"id":"https://openalex.org/keywords/sensitivity","display_name":"Sensitivity (control systems)","score":0.4417000114917755},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4383000135421753},{"id":"https://openalex.org/keywords/position","display_name":"Position (finance)","score":0.42559999227523804},{"id":"https://openalex.org/keywords/fraction","display_name":"Fraction (chemistry)","score":0.38179999589920044}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5909000039100647},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5394999980926514},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4763999879360199},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.4417000114917755},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4383000135421753},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.42559999227523804},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.38179999589920044},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.37950000166893005},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.3720000088214874},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.3700999915599823},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3336000144481659},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.31470000743865967},{"id":"https://openalex.org/C65856478","wikidata":"https://www.wikidata.org/wiki/Q3991682","display_name":"Attack model","level":2,"score":0.31380000710487366},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.2806999981403351},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09070","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09070","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09070","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09070","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.8269010186195374,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Many":[0],"jailbreak":[1,32,62,68,133,261],"attack":[2,6,118,187],"research":[3],"papers":[4,33],"report":[5],"success":[7],"rates":[8],"for":[9,55,132,259],"a":[10,124],"limited":[11],"number":[12],"of":[13,22,43,98,116,167,182,232],"parameter":[14,23],"settings,":[15],"even":[16],"though":[17],"there":[18],"are":[19,34,52],"many":[20],"combinations":[21],"settings":[24],"that":[25,100,106,237],"could":[26],"be":[27],"used.":[28],"Further,":[29],"when":[30],"new":[31,130,256],"released,":[35],"they":[36],"often":[37],"benchmark":[38],"results":[39],"against":[40],"single":[41,125],"configurations":[42],"existing":[44],"attacks.":[45,66],"This":[46],"position":[47],"paper":[48],"argues":[49],"such":[50],"practices":[51],"fundamentally":[53],"insufficient":[54],"characterising":[56],"the":[57,92,110,117,135,148,154,158,164,180,196,220,227,255],"threat":[58],"posed":[59],"by":[60,122],"parameterised":[61,260],"attacks,":[63],"and":[64,83,113,140,204,213,244],"comparing":[65],"Most":[67],"attacks":[69],"expose":[70],"multiple":[71],"internal":[72],"parameters,":[73],"system":[74],"prompt":[75],"templates,":[76],"conversation":[77],"rounds,":[78],"cipher":[79],"dispersion,":[80],"teaching":[81],"shots,":[82],"ASR":[84,151,156,201,243],"varies":[85],"substantially":[86],"across":[87,109,157,173,189],"these":[88,183],"parameters.":[89],"Reporting":[90],"only":[91],"best-case":[93],"configuration":[94],"discards":[95],"two":[96,129,186],"pieces":[97],"information":[99],"defenders":[101],"genuinely":[102],"need:":[103],"how":[104,114,146],"typical":[105],"performance":[107],"is":[108,120,163],"variant":[111,160,222,246],"space,":[112,161],"much":[115],"surface":[119],"missed":[121],"selecting":[123],"variant.":[126],"We":[127,177,235],"propose":[128],"measures":[131,184],"attacks:":[134],"Variant":[136],"Sensitivity":[137],"Measure":[138],"(VSM)":[139],"Union":[141],"Coverage":[142],"(UC).":[143],"VSM":[144,241],"quantifies":[145],"far":[147],"best":[149,197,221],"reported":[150],"deviates":[152],"from":[153],"mean":[155],"tested":[159,175],"UC":[162,209],"total":[165],"fraction":[166],"prompts":[168],"resulting":[169],"in":[170],"unsafe":[171],"responses":[172],"all":[174],"configurations.":[176],"empirically":[178],"demonstrate":[179],"importance":[181],"using":[185],"families":[188],"three":[190],"open-source":[191],"target":[192],"models.":[193],"For":[194,216],"PAIR,":[195],"template":[198],"reaches":[199,223],"69%":[200],"on":[202,206,218],"Mistral-7B":[203],"75%":[205],"Qwen3-0.6B,":[207],"while":[208],"rises":[210],"to":[211],"88%":[212],"93%,":[214],"respectively.":[215],"bijection":[217],"Mistral-7B,":[219],"81%":[224],"ASR,":[225],"but":[226],"36-variant":[228],"union":[229],"covers":[230],"100%":[231],"HarmBench-100":[233],"prompts.":[234],"argue":[236],"distributional":[238],"reporting,":[239],"publishing":[240],"alongside":[242],"enumerating":[245],"coverage":[247],"as":[248,250],"fully":[249],"compute":[251],"allows,":[252],"should":[253],"become":[254],"minimum":[257],"standard":[258],"evaluation.":[262]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-13T00:00:00"}