{"id":"https://openalex.org/W4415539073","doi":"https://doi.org/10.1145/3746027.3754561","title":"JPS: Jailbreak Multimodal Large Language Models with Collaborative Visual Perturbation and Textual Steering","display_name":"JPS: Jailbreak Multimodal Large Language Models with Collaborative Visual Perturbation and Textual Steering","publication_year":2025,"publication_date":"2025-10-25","ids":{"openalex":"https://openalex.org/W4415539073","doi":"https://doi.org/10.1145/3746027.3754561"},"language":"en","primary_location":{"id":"doi:10.1145/3746027.3754561","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746027.3754561","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3746027.3754561","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010074373","display_name":"Renmiao Chen","orcid":"https://orcid.org/0009-0001-1213-0916"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Renmiao Chen","raw_affiliation_strings":["CoAI, DCST, Tsinghua University, Beijing, China and Zhipu AI, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-1213-0916","affiliations":[{"raw_affiliation_string":"CoAI, DCST, Tsinghua University, Beijing, China and Zhipu AI, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028351629","display_name":"Shiyao Cui","orcid":"https://orcid.org/0009-0003-2333-1064"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shiyao Cui","raw_affiliation_strings":["CoAI group, DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-2333-1064","affiliations":[{"raw_affiliation_string":"CoAI group, DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103874857","display_name":"Xuancheng Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuancheng Huang","raw_affiliation_strings":["Zhipu.AI, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-4557-7546","affiliations":[{"raw_affiliation_string":"Zhipu.AI, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030668126","display_name":"Chengwei Pan","orcid":"https://orcid.org/0000-0003-0497-7903"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengwei Pan","raw_affiliation_strings":["Beihang Uinveristy, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0497-7903","affiliations":[{"raw_affiliation_string":"Beihang Uinveristy, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114334984","display_name":"Victor Shea-Jay Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Victor Shea-Jay Huang","raw_affiliation_strings":["Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-0614-5699","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111837778","display_name":"Qinglin Zhang","orcid":"https://orcid.org/0009-0009-5270-3801"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"QingLin Zhang","raw_affiliation_strings":["CoAI group, DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-5270-3801","affiliations":[{"raw_affiliation_string":"CoAI group, DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109461795","display_name":"Xuan Ouyang","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuan Ouyang","raw_affiliation_strings":["CoAI group, DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0005-4115-7340","affiliations":[{"raw_affiliation_string":"CoAI group, DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102245113","display_name":"Zhexin Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhexin Zhang","raw_affiliation_strings":["CoAI group, DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-9601-3991","affiliations":[{"raw_affiliation_string":"CoAI group, DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085094109","display_name":"Hongning Wang","orcid":"https://orcid.org/0000-0002-6524-9195"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongning Wang","raw_affiliation_strings":["CoAI group, DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6524-9195","affiliations":[{"raw_affiliation_string":"CoAI group, DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044042138","display_name":"Minlie Huang","orcid":"https://orcid.org/0000-0001-7111-1849"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Minlie Huang","raw_affiliation_strings":["CoAI group, DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-7111-1849","affiliations":[{"raw_affiliation_string":"CoAI group, DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"11756","last_page":"11765"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9495000243186951,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9495000243186951,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.941100001335144,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.9266999959945679,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.7512000203132629},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.39160001277923584},{"id":"https://openalex.org/keywords/visual-language","display_name":"Visual language","score":0.3540000021457672},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.33559998869895935},{"id":"https://openalex.org/keywords/perturbation","display_name":"Perturbation (astronomy)","score":0.3296000063419342},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.321399986743927}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.7512000203132629},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7128999829292297},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40849998593330383},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3962000012397766},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.39160001277923584},{"id":"https://openalex.org/C2780878386","wikidata":"https://www.wikidata.org/wiki/Q1659648","display_name":"Visual language","level":2,"score":0.3540000021457672},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.33559998869895935},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.3296000063419342},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.321399986743927},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.31700000166893005},{"id":"https://openalex.org/C140547941","wikidata":"https://www.wikidata.org/wiki/Q7797194","display_name":"Threat model","level":2,"score":0.3037000000476837},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.2809999883174896},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26570001244544983},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.25119999051094055}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3746027.3754561","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746027.3754561","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2508.05087","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.05087","pdf_url":"https://arxiv.org/pdf/2508.05087","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3746027.3754561","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746027.3754561","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W4385374425","https://openalex.org/W4387635776","https://openalex.org/W4388585701","https://openalex.org/W4391590954","https://openalex.org/W4399455069","https://openalex.org/W4400375311","https://openalex.org/W4402581611","https://openalex.org/W4403853618","https://openalex.org/W4403878024","https://openalex.org/W4404450583","https://openalex.org/W4404988906","https://openalex.org/W4405033163","https://openalex.org/W6929434350"],"related_works":[],"abstract_inverted_index":{"Jailbreak":[0,60],"attacks":[1],"against":[2],"multimodal":[3],"large":[4],"language":[5],"Models":[6],"(MLLMs)":[7],"are":[8,166],"a":[9,98,139,147],"significant":[10],"research":[11,14],"focus.":[12],"Current":[13],"predominantly":[15],"focuses":[16],"on":[17],"maximizing":[18],"attack":[19,126],"success":[20],"rate":[21],"(ASR),":[22],"often":[23],"overlooking":[24],"whether":[25],"the":[26,31,107,123,130],"generated":[27],"responses":[28,105],"actually":[29],"fulfill":[30],"attacker's":[32],"malicious":[33],"intent.":[34,109],"This":[35,171],"oversight":[36],"frequently":[37],"leads":[38],"to":[39,101],"low-quality":[40],"outputs":[41],"that,":[42],"while":[43],"successful":[44],"in":[45,150],"bypassing":[46],"safety":[47,90],"filters,":[48],"lack":[49],"substantial":[50],"harmful":[51],"content.":[52],"To":[53,121],"address":[54],"this":[55],"gap,":[56],"we":[57,128],"propose":[58,129],"JPS,":[59],"MLLMs":[61,157],"with":[62,160],"collaborative":[63],"visual":[64,75,111],"Perturbation":[65],"and":[66,77,112,153,158],"textual":[67,113],"Steering,":[68],"which":[69],"achieves":[70],"jailbreaks":[71],"via":[72,97],"corporation":[73],"of":[74,125],"image":[76,86],"textually":[78],"steering":[79],"prompt.":[80],"Specifically,":[81],"JPS":[82,145],"utilizes":[83],"target-guided":[84],"adversarial":[85],"perturbations":[87],"for":[88,118],"effective":[89],"bypass,":[91],"complemented":[92],"by":[93],"''steering":[94],"prompt''":[95],"optimized":[96],"multi-agent":[99],"system":[100],"specifically":[102],"guide":[103],"LLM":[104],"fulfilling":[106],"attackers'":[108],"These":[110],"components":[114],"undergo":[115],"iterative":[116],"co-optimization":[117],"enhanced":[119],"performance.":[120],"evaluate":[122],"quality":[124],"outcomes,":[127],"Malicious":[131],"Intent":[132],"Fulfillment":[133],"Rate":[134],"(MIFR)":[135],"metric,":[136],"assessed":[137],"using":[138],"Reasoning-LLM-based":[140],"evaluator.":[141],"Our":[142],"experiments":[143],"show":[144],"sets":[146],"new":[148],"state-of-the-art":[149],"both":[151],"ASR":[152],"MIFR":[154],"across":[155],"various":[156],"benchmarks,":[159],"analyses":[161],"confirming":[162],"its":[163],"efficacy.":[164],"Codes":[165],"available":[167],"at":[168],"https://github.com/thu-coai/JPS":[169],"Warning:":[170],"paper":[172],"contains":[173],"potentially":[174],"sensitive":[175],"contents.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-25T00:00:00"}