{"id":"https://openalex.org/W4411799673","doi":"https://doi.org/10.1109/tifs.2025.3583249","title":"Jailbreak Vision Language Models via Bi-Modal Adversarial Prompt","display_name":"Jailbreak Vision Language Models via Bi-Modal Adversarial Prompt","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4411799673","doi":"https://doi.org/10.1109/tifs.2025.3583249"},"language":"en","primary_location":{"id":"doi:10.1109/tifs.2025.3583249","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tifs.2025.3583249","pdf_url":null,"source":{"id":"https://openalex.org/S61310614","display_name":"IEEE Transactions on Information Forensics and Security","issn_l":"1556-6013","issn":["1556-6013","1556-6021"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Information Forensics and Security","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020945039","display_name":"Zonghao Ying","orcid":"https://orcid.org/0009-0007-7393-7362"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zonghao Ying","raw_affiliation_strings":["SKLCCSE, School of Artificial Intelligence, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-7393-7362","affiliations":[{"raw_affiliation_string":"SKLCCSE, School of Artificial Intelligence, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014870180","display_name":"Aishan Liu","orcid":"https://orcid.org/0000-0002-4224-1318"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Aishan Liu","raw_affiliation_strings":["SKLCCSE, School of Computer Science and Engineering, Beihang University, Beijing, China","School of Computer Science and Engineering, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4224-1318","affiliations":[{"raw_affiliation_string":"SKLCCSE, School of Computer Science and Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"School of Computer Science and Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023476317","display_name":"Tianyuan Zhang","orcid":"https://orcid.org/0000-0001-9874-6828"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianyuan Zhang","raw_affiliation_strings":["SKLCCSE, School of Computer Science and Engineering, Beihang University, Beijing, China","School of Computer Science and Engineering, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9874-6828","affiliations":[{"raw_affiliation_string":"SKLCCSE, School of Computer Science and Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"School of Computer Science and Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103182703","display_name":"Zhengmin Yu","orcid":"https://orcid.org/0000-0002-0294-414X"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhengmin Yu","raw_affiliation_strings":["School of Computer Science, Fudan University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081392445","display_name":"Siyuan Liang","orcid":"https://orcid.org/0000-0002-6154-0233"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Siyuan Liang","raw_affiliation_strings":["School of Computing, National University of Singapore, Queenstown, Singapore","School of Computing, National University of Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-6154-0233","affiliations":[{"raw_affiliation_string":"School of Computing, National University of Singapore, Queenstown, Singapore","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"School of Computing, National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024067284","display_name":"Xianglong Liu","orcid":"https://orcid.org/0000-0001-8425-4195"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xianglong Liu","raw_affiliation_strings":["SKLCCSE, School of Computer Science and Engineering, Beihang University, Beijing, China","School of Computer Science and Engineering, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8425-4195","affiliations":[{"raw_affiliation_string":"SKLCCSE, School of Computer Science and Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"School of Computer Science and Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074103823","display_name":"Dacheng Tao","orcid":"https://orcid.org/0000-0001-7225-5449"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Dacheng Tao","raw_affiliation_strings":["College of Computing and Data Science, Nanyang Technological University, Jurong West, Singapore","College of Computing & Data Science, Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-7225-5449","affiliations":[{"raw_affiliation_string":"College of Computing and Data Science, Nanyang Technological University, Jurong West, Singapore","institution_ids":["https://openalex.org/I172675005"]},{"raw_affiliation_string":"College of Computing & Data Science, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":22.1721,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.99355556,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"20","issue":null,"first_page":"7153","last_page":"7165"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12034","display_name":"Digital and Cyber Forensics","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9628999829292297,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8363032937049866},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.6453397274017334},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6443020105361938},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4898819327354431},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4106842875480652},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3949960768222809},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3447904586791992}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8363032937049866},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.6453397274017334},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6443020105361938},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4898819327354431},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4106842875480652},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3949960768222809},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3447904586791992},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tifs.2025.3583249","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tifs.2025.3583249","pdf_url":null,"source":{"id":"https://openalex.org/S61310614","display_name":"IEEE Transactions on Information Forensics and Security","issn_l":"1556-6013","issn":["1556-6013","1556-6021"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Information Forensics and Security","raw_type":"journal-article"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/217829","is_oa":false,"landing_page_url":"https://hdl.handle.net/10356/217829","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3834788717","display_name":null,"funder_award_id":"Grant. 62206009","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G489730147","display_name":null,"funder_award_id":"62206009","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":42,"referenced_works":["https://openalex.org/W1550291595","https://openalex.org/W2122374500","https://openalex.org/W2171770604","https://openalex.org/W2986378528","https://openalex.org/W3203247393","https://openalex.org/W4293846201","https://openalex.org/W4312605942","https://openalex.org/W4322718191","https://openalex.org/W4366850747","https://openalex.org/W4385374425","https://openalex.org/W4386907292","https://openalex.org/W4386977422","https://openalex.org/W4387355767","https://openalex.org/W4387436631","https://openalex.org/W4387559087","https://openalex.org/W4387561041","https://openalex.org/W4387635776","https://openalex.org/W4388482181","https://openalex.org/W4388482289","https://openalex.org/W4389217677","https://openalex.org/W4389364218","https://openalex.org/W4389364443","https://openalex.org/W4389617257","https://openalex.org/W4389713766","https://openalex.org/W4390190425","https://openalex.org/W4390660439","https://openalex.org/W4391159098","https://openalex.org/W4391506087","https://openalex.org/W4391590954","https://openalex.org/W4391833808","https://openalex.org/W4392934624","https://openalex.org/W4393157467","https://openalex.org/W4402670859","https://openalex.org/W4402671875","https://openalex.org/W4402727057","https://openalex.org/W4402727490","https://openalex.org/W4403345328","https://openalex.org/W4404575054","https://openalex.org/W4404783010","https://openalex.org/W4409348010","https://openalex.org/W4411119795","https://openalex.org/W4412073525"],"related_works":["https://openalex.org/W2502115930","https://openalex.org/W2482350142","https://openalex.org/W4246396837","https://openalex.org/W3126451824","https://openalex.org/W1561927205","https://openalex.org/W3191453585","https://openalex.org/W4297672492","https://openalex.org/W4310988119","https://openalex.org/W4285226279","https://openalex.org/W4288019534"],"abstract_inverted_index":{"In":[0,130],"the":[1,28,36,65,107,120,156,191],"realm":[2],"of":[3,158,193],"large":[4,135,179],"vision":[5],"language":[6,136],"models":[7,48],"(LVLMs),":[8],"jailbreak":[9,140],"attacks":[10,195],"serve":[11],"as":[12,201],"a":[13,93,134,151],"red-teaming":[14],"approach":[15],"to":[16,112,115,138,146],"bypass":[17],"guardrails":[18],"and":[19,52,77,100,142,168,203],"uncover":[20],"safety":[21],"implications.":[22],"Existing":[23],"jailbreaks":[24,73],"predominantly":[25],"focus":[26],"on":[27,165,186,196],"visual":[29,33,51,78],"modality,":[30],"perturbing":[31],"solely":[32],"inputs":[34],"in":[35,88,182],"prompt":[37,110],"for":[38,56],"attacks.":[39],"However,":[40],"they":[41],"fall":[42],"short":[43],"when":[44],"confronted":[45],"with":[46,126],"aligned":[47],"that":[49,106,171],"fuse":[50],"textual":[53,76,124,148],"features":[54],"simultaneously":[55],"generation.":[57],"To":[58,154],"address":[59],"this":[60,62],"limitation,":[61],"paper":[63],"introduces":[64],"Bi-Modal":[66],"Adversarial":[67],"Prompt":[68],"Attack":[69],"(BAP),":[70],"which":[71],"executes":[72],"by":[74,92,178],"optimizing":[75],"prompts":[79,125,149],"cohesively.":[80],"Initially,":[81],"we":[82,122,132,161,189],"adversarially":[83],"embed":[84],"universally":[85],"adversarial":[86,108],"perturbations":[87],"an":[89],"image,":[90,121],"guided":[91],"few-shot":[94],"query-agnostic":[95],"corpus":[96],"(e.g.,":[97],"affirmative":[98],"prefixes":[99],"negative":[101],"inhibitions).":[102],"This":[103],"process":[104],"ensures":[105],"image":[109],"LVLMs":[111],"respond":[113],"positively":[114],"harmful":[116,128],"queries.":[117],"Subsequently,":[118],"leveraging":[119],"optimize":[123],"specific":[127],"intent.":[129],"particular,":[131],"utilize":[133],"model":[137],"analyze":[139],"failures":[141],"employ":[143],"chain-of-thought":[144],"reasoning":[145],"refine":[147],"through":[150],"feedback-iteration":[152],"manner.":[153],"validate":[155],"efficacy":[157],"our":[159,172,194],"approach,":[160],"conducted":[162],"extensive":[163],"evaluations":[164],"various":[166],"datasets":[167],"LVLMs,":[169,199],"demonstrating":[170],"BAP":[173],"significantly":[174],"outperforms":[175],"other":[176],"methods":[177],"margins":[180],"(+29.03%":[181],"attack":[183],"success":[184],"rate":[185],"average).":[187],"Additionally,":[188],"showcase":[190],"potential":[192],"black-box":[197],"commercial":[198],"such":[200],"GPT-4o":[202],"Gemini.":[204]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":7}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}