{"id":"https://openalex.org/W7165959732","doi":"https://doi.org/10.48550/arxiv.2606.25487","title":"How Reliable Is Your Jailbreak Judge? Calibration and Adversarial Robustness of Automated ASR Scoring","display_name":"How Reliable Is Your Jailbreak Judge? Calibration and Adversarial Robustness of Automated ASR Scoring","publication_year":2026,"publication_date":"2026-06-24","ids":{"openalex":"https://openalex.org/W7165959732","doi":"https://doi.org/10.48550/arxiv.2606.25487"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.25487","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25487","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.25487","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139386497","display_name":"Yang Gao","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107404","display_name":"Clinical Research Solutions","ror":"https://ror.org/01m2t4z13","country_code":"US","type":"facility","lineage":["https://openalex.org/I4210107404"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Gao, Yang","raw_affiliation_strings":["Veyon Solutions"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Veyon Solutions","institution_ids":["https://openalex.org/I4210107404"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5139386497"],"corresponding_institution_ids":["https://openalex.org/I4210107404"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5335000157356262,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5335000157356262,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.0763000026345253,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12034","display_name":"Digital and Cyber Forensics","score":0.05480000004172325,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.6656000018119812},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.548799991607666},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.498199999332428},{"id":"https://openalex.org/keywords/harm","display_name":"Harm","score":0.4943999946117401},{"id":"https://openalex.org/keywords/false-positive-paradox","display_name":"False positive paradox","score":0.46209999918937683},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.45980000495910645},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.40400001406669617}],"concepts":[{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.6656000018119812},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.659600019454956},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.548799991607666},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5085999965667725},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.498199999332428},{"id":"https://openalex.org/C2777363581","wikidata":"https://www.wikidata.org/wiki/Q15098235","display_name":"Harm","level":2,"score":0.4943999946117401},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.46209999918937683},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.45980000495910645},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.40400001406669617},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40310001373291016},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.37070000171661377},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.3517000079154968},{"id":"https://openalex.org/C2989486834","wikidata":"https://www.wikidata.org/wiki/Q3808900","display_name":"True positive rate","level":2,"score":0.34880000352859497},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3384999930858612},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3384999930858612},{"id":"https://openalex.org/C52001869","wikidata":"https://www.wikidata.org/wiki/Q812530","display_name":"Naive Bayes classifier","level":3,"score":0.3328999876976013},{"id":"https://openalex.org/C169087156","wikidata":"https://www.wikidata.org/wiki/Q2131593","display_name":"Framing (construction)","level":2,"score":0.29120001196861267},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2777999937534332},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25690001249313354},{"id":"https://openalex.org/C65856478","wikidata":"https://www.wikidata.org/wiki/Q3991682","display_name":"Attack model","level":2,"score":0.25450000166893005},{"id":"https://openalex.org/C2987098735","wikidata":"https://www.wikidata.org/wiki/Q3808900","display_name":"Recall rate","level":2,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.25487","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25487","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.25487","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.25487","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.8147739768028259,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Almost":[0],"every":[1,141,213],"paper":[2],"on":[3,115,179,242,257],"LLM":[4],"jailbreaks":[5],"and":[6,14,71,135,145,150,227,244,255,267],"prompt":[7],"injection":[8],"reports":[9],"an":[10,24,269],"attack-success":[11],"rate":[12],"(ASR),":[13],"that":[15,129,250],"number":[16],"is":[17,45,277],"assigned":[18],"not":[19],"by":[20,23],"people":[21],"but":[22,99,174],"automated":[25],"judge:":[26],"either":[27],"a":[28,36,151,175,199,225,258],"safety":[29],"classifier":[30,58,84,166],"trained":[31],"for":[32,157,264],"the":[33,56,63,107,131,148,207,210,221,273],"task,":[34],"or":[35],"general":[37],"chat":[38],"model":[39],"prompted":[40],"to":[41,97,104,162,195],"grade.":[42],"The":[43,75,82,120,164],"judge":[44,65,117,253,265],"rarely":[46],"checked.":[47],"We":[48,248],"check":[49,271],"it.":[50],"Using":[51],"596":[52],"human-labeled":[53,259],"completions":[54],"from":[55,234],"HarmBench":[57],"validation":[59],"set,":[60],"we":[61],"compare":[62],"two":[64,76,121],"families":[66,77,122],"against":[67],"human":[68],"majority":[69],"votes":[70],"then":[72],"attack":[73,178],"them.":[74,119],"fail":[78],"in":[79,126],"opposite":[80],"ways.":[81],"dedicated":[83,165],"over-flags":[85],"(precision":[86],"0.835,":[87],"recall":[88,102,256],"0.974);":[89],"three":[90],"different":[91,112],"LLM-as-judges":[92],"keep":[93],"high":[94],"precision":[95,254],"(0.81":[96],"0.94)":[98],"show":[100],"erratic":[101],"(0.06":[103],"0.65),":[105],"so":[106],"same":[108],"responses":[109],"produce":[110],"very":[111],"ASR":[113,232,262],"depending":[114],"which":[116],"scores":[118],"also":[123],"differ":[124],"sharply":[125],"robustness.":[127],"Wrappers":[128],"leave":[130,209],"harmful":[132,222],"text":[133],"untouched":[134],"only":[136],"add":[137],"benign":[138],"framing":[139],"flip":[140],"LLM-judge":[142],"between":[143],"57%":[144],"100%":[146],"of":[147,159,185,190,215,230,272],"time,":[149],"single":[152],"prepended":[153],"refusal":[154],"sentence":[155],"accounts":[156],"much":[158],"this":[160],"(39%":[161],"88%).":[163],"resists":[167],"these":[168],"surface":[169],"attacks":[170,208],"(at":[171],"most":[172],"6.7%),":[173],"white-box":[176],"GCG":[177],"its":[180],"open":[181],"weights":[182],"flips":[183,218],"70%":[184],"confident":[186],"true":[187],"positives":[188],"(21":[189],"30;":[191],"95%":[192],"CI":[193],"54":[194],"86%)":[196],"even":[197],"at":[198],"small":[200],"optimization":[201],"budget.":[202],"A":[203],"two-annotator":[204],"audit":[205],"confirms":[206],"harm":[211],"intact:":[212],"one":[214],"80":[216],"sampled":[217],"still":[219],"contained":[220],"content.":[223],"Because":[224],"large":[226],"growing":[228],"share":[229],"reported":[231],"comes":[233],"LLM-judges,":[235],"many":[236],"such":[237],"numbers":[238],"are":[239],"unreliable":[240],"both":[241],"average":[243],"under":[245],"deliberate":[246],"pressure.":[247],"recommend":[249],"papers":[251],"report":[252,261],"slice,":[260],"corrected":[263],"precision,":[266],"include":[268],"adversarial":[270],"judge.":[274],"Our":[275],"code":[276],"released.":[278]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-26T00:00:00"}