{"id":"https://openalex.org/W7161269413","doi":"https://doi.org/10.48550/arxiv.2605.14418","title":"The Great Pretender: A Stochasticity Problem in LLM Jailbreak","display_name":"The Great Pretender: A Stochasticity Problem in LLM Jailbreak","publication_year":2026,"publication_date":"2026-05-14","ids":{"openalex":"https://openalex.org/W7161269413","doi":"https://doi.org/10.48550/arxiv.2605.14418"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.14418","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14418","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.14418","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047495482","display_name":"Jean-Philippe Monteuuis","orcid":"https://orcid.org/0000-0001-7712-1132"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Monteuuis, Jean-Philippe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136257619","display_name":"Cong Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Cong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136258210","display_name":"Jonathan Petit","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Petit, Jonathan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.22439999878406525,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.22439999878406525,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.1088000014424324,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.08290000259876251,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6283000111579895},{"id":"https://openalex.org/keywords/wonder","display_name":"Wonder","score":0.6055999994277954},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.517300009727478},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.3968000113964081},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.3619999885559082},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.2896000146865845}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6283000111579895},{"id":"https://openalex.org/C2780573756","wikidata":"https://www.wikidata.org/wiki/Q977433","display_name":"Wonder","level":2,"score":0.6055999994277954},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.605400025844574},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.517300009727478},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.3968000113964081},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3619999885559082},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32350000739097595},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.30090001225471497},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C2984949393","wikidata":"https://www.wikidata.org/wiki/Q1260241","display_name":"Drop out","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2782999873161316},{"id":"https://openalex.org/C65856478","wikidata":"https://www.wikidata.org/wiki/Q3991682","display_name":"Attack model","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2628999948501587}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.14418","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14418","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.14418","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.14418","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"\"Oh-Oh,":[0],"yes,":[1],"I'm":[2,8],"the":[3,20,23,59,63,76,85,91,113,132,175,186],"great":[4],"pretender.":[5],"Pretending":[6],"that":[7,232],"doing":[9],"well.":[10],"My":[11],"need":[12],"is":[13,139],"such,":[14],"I":[15],"pretend":[16],"too":[17],"much...\"":[18],"summarizes":[19],"state":[21],"in":[22,62],"area":[24],"of":[25,102,188,240,274],"jailbreak":[26,77,87,137,162,204,248,266],"creation":[27],"and":[28,152,209,211,220,225,268],"evaluation.":[29],"You":[30,73],"find":[31],"this":[32,53,182,272],"method":[33,54],"to":[34,125,242,251],"generate":[35,75],"adversarial":[36],"attacks":[37],"proposed":[38],"by":[39],"a":[40,95,118,141,160,170,217,247],"reputable":[41],"institution":[42],"(e.g.,":[43],"BoN":[44],"from":[45,49],"Anthropic":[46],"or":[47],"Crescendo":[48],"Microsoft":[50],"Research).":[51],"However,":[52,84],"does":[55,164],"not":[56,140,165,190],"deliver":[57],"on":[58,112,173,253],"promise":[60],"claimed":[61],"paper":[64],"despite":[65,105],"having":[66,106],"top":[67],"ASR":[68,109,146,238],"scores":[69],"against":[70,79,90,169],"industry-grade":[71],"LLMs.":[72],"successfully":[74],"prompts":[78,176],"your":[80],"target":[81,92,171],"(open)":[82],"model.":[83],"generated":[86],"prompt":[88,163,249],"works":[89],"model":[93,116,172],"with":[94],"50%":[96],"consecutive":[97],"success":[98],"rate":[99],"(5":[100],"out":[101],"10":[103],"attempts)":[104],"an":[107,233,237],"80%":[108],"(on":[110],"paper)":[111],"latest":[114],"closed-source":[115],"(with":[117],"guardrail":[119],"system)!":[120],"This":[121],"observation":[122],"leads":[123],"us":[124],"think.":[126],"First,":[127],"Attack":[128],"Success":[129],"Rate":[130],"(ASR),":[131],"primary":[133],"metric":[134,219],"for":[135],"LLM":[136],"benchmarking,":[138],"stable":[142],"quantity.":[143],"Second,":[144],"published":[145],"numbers":[147],"are":[148],"therefore":[149],"systematically":[150],"inflated":[151],"incomparable":[153],"across":[154],"papers.":[155],"Therefore,":[156],"we":[157,184,215],"wonder":[158],"\"Why":[159],"successful":[161],"perform":[166],"consistently":[167],"well":[168],"which":[174],"have":[177,236],"been":[178],"optimized?\".":[179],"To":[180],"answer":[181],"question,":[183],"study":[185],"impact":[187],"stochasticity":[189],"only":[191],"during":[192,197],"attack":[193,198,234,260],"evaluation":[194,201,228],"but":[195],"also":[196],"generation.":[199],"Our":[200,227],"includes":[202],"several":[203],"attacks,":[205],"models":[206],"(different":[207],"sizes":[208],"providers),":[210],"judges.":[212],"In":[213],"addition,":[214],"propose":[216],"new":[218,222],"two":[221],"frameworks":[223],"(CAS-eval":[224],"CAS-gen).":[226],"framework,":[229],"CAS-eval,":[230],"shows":[231],"can":[235],"drop":[239],"up":[241],"30":[243,275],"percentage":[244,276],"points":[245],"when":[246],"needs":[250],"succeed":[252],"more":[254],"than":[255],"one":[256],"attempt.":[257],"Thankfully,":[258],"our":[259],"generation":[261],"framework":[262],"(CAS-gen)":[263],"improves":[264],"previous":[265],"methods":[267],"helps":[269],"them":[270],"recover":[271],"loss":[273],"points!":[277]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-16T00:00:00"}