{"id":"https://openalex.org/W7202281374","doi":"https://doi.org/10.48550/arxiv.2608.10959","title":"Once Poisoned, Arbitrarily Controlled: A Programmable Backdoor in VLMs","display_name":"Once Poisoned, Arbitrarily Controlled: A Programmable Backdoor in VLMs","publication_year":2026,"publication_date":"2026-08-11","ids":{"openalex":"https://openalex.org/W7202281374","doi":"https://doi.org/10.48550/arxiv.2608.10959"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.10959","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.10959","pdf_url":"https://arxiv.org/pdf/2608.10959","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.10959","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5147026213","display_name":"Tao Lin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123993160","display_name":"Gaojie Jin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"Jin, Gaojie","raw_affiliation_strings":["Department of Artificial Intelligence , University of Macau , Macau , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence , University of Macau , Macau , China","institution_ids":["https://openalex.org/I204512498"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026684797","display_name":"Zongxin Liu","orcid":"https://orcid.org/0000-0002-4127-1806","observed_orcids":["https://orcid.org/0000-0002-4127-1806"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zongxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146972458","display_name":"Peng Wu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Peng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5125772558","display_name":"Lijia Yu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Lijia","raw_affiliation_strings":["Institute of AI for Industries , Chinese Academy of Sciences , Nanjing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of AI for Industries , Chinese Academy of Sciences , Nanjing , China","institution_ids":["https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.8799999952316284},{"id":"https://openalex.org/keywords/backdoor-attacks","display_name":"backdoor attacks","score":0.9739999771118164},{"id":"https://openalex.org/keywords/data-poisoning-attacks","display_name":"data poisoning","score":0.8399999737739563},{"id":"https://openalex.org/keywords/backdoor-defense","display_name":"backdoor defense","score":0.6970000267028809},{"id":"https://openalex.org/keywords/adversarial-robustness","display_name":"adversarial robustness","score":0.5120000243186951}],"concepts":[{"id":"https://openalex.org/C2781045450","wikidata":"https://www.wikidata.org/wiki/Q254569","display_name":"Backdoor","level":2,"score":0.9876999855041504},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6474000215530396},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.47450000047683716},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.4047999978065491},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.39489999413490295},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39259999990463257},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.37299999594688416},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.31769999861717224},{"id":"https://openalex.org/C73301696","wikidata":"https://www.wikidata.org/wiki/Q5469984","display_name":"Formalism (music)","level":3,"score":0.3167000114917755},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.30410000681877136},{"id":"https://openalex.org/C193856179","wikidata":"https://www.wikidata.org/wiki/Q5251100","display_name":"Defeasible estate","level":2,"score":0.28220000863075256},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.25380000472068787}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.10959","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.10959","pdf_url":"https://arxiv.org/pdf/2608.10959","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.10959","doi":"https://doi.org/10.48550/arxiv.2608.10959","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.10959","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.10959","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.10959","pdf_url":"https://arxiv.org/pdf/2608.10959","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7202281374.pdf","grobid_xml":"https://content.openalex.org/works/W7202281374.grobid-xml"},"referenced_works_count":15,"referenced_works":["https://openalex.org/W2570685808","https://openalex.org/W2748789698","https://openalex.org/W2993846550","https://openalex.org/W3121478722","https://openalex.org/W3140988292","https://openalex.org/W3166049810","https://openalex.org/W3175052694","https://openalex.org/W3176270593","https://openalex.org/W4319653550","https://openalex.org/W4366850747","https://openalex.org/W4383472654","https://openalex.org/W4387723654","https://openalex.org/W4403883660","https://openalex.org/W4416780398","https://openalex.org/W4417227621"],"related_works":["https://openalex.org/W7138190816","https://openalex.org/W7160327541","https://openalex.org/W4416953404","https://openalex.org/W7153305462","https://openalex.org/W4417127580","https://openalex.org/W4404563144","https://openalex.org/W7106662527","https://openalex.org/W7161906593","https://openalex.org/W7129326401","https://openalex.org/W7129381619"],"abstract_inverted_index":{"Existing":[0],"vision-language":[1],"model":[2,105,194],"(VLM)":[3],"backdoors":[4],"are":[5],"usually":[6],"treated":[7],"as":[8,142],"static":[9],"vulnerabilities:":[10],"one-to-one":[11],"and":[12,62,196],"N-to-N":[13],"attacks":[14],"bind":[15],"one":[16],"or":[17,147],"more":[18],"triggers":[19,66,157],"to":[20,53,106,112,136,162],"a":[21,38,44,48,99,114,121,126,137,144,148],"finite":[22],"set":[23],"of":[24,86],"targets":[25],"before":[26],"victim":[27],"training.":[28],"This":[29],"assumption":[30],"substantially":[31],"underestimates":[32],"the":[33,72,91,104,159,168,174],"threat.":[34],"We":[35],"show":[36,182],"that":[37,183],"single":[39],"poisoning":[40,101],"phase":[41],"can":[42],"implant":[43],"programmable":[45],"backdoor":[46,123,202],"into":[47,153],"VLM,":[49],"allowing":[50],"an":[51],"attacker":[52],"choose":[54],"previously":[55],"unseen":[56,177],"target-caption":[57],"semantics":[58],"at":[59],"inference":[60],"time":[61],"synthesize":[63],"corresponding":[64],"stealthy":[65,138],"on":[67],"demand.":[68],"Unlike":[69],"fixed-mapping":[70],"attacks,":[71],"proposed":[73],"any-to-any":[74,188],"caption-control":[75,189],"paradigm":[76],"decouples":[77],"post-training":[78],"target":[79,87,134,170,175],"selection":[80],"from":[81],"poisoning,":[82],"enabling":[83],"dynamic":[84],"control":[85],"captions":[88],"without":[89],"retraining":[90],"VLM.":[92],"Our":[93],"method":[94,130],"has":[95],"two":[96],"components.":[97],"First,":[98],"heuristic":[100],"strategy":[102],"exposes":[103],"diverse":[107],"trigger-caption":[108],"pairs,":[109],"encouraging":[110],"it":[111],"learn":[113],"general":[115],"trigger-as-instruction":[116],"rule":[117],"rather":[118],"than":[119],"memorize":[120],"specific":[122],"pattern.":[124],"Second,":[125],"feature-space":[127],"trigger":[128],"steganography":[129],"maps":[131],"any":[132],"attacker-specified":[133],"caption":[135],"visual":[139],"trigger,":[140],"implemented":[141],"either":[143],"norm-controlled":[145],"perturbation":[146],"non-semantic":[149],"patch.":[150],"Once":[151],"inserted":[152],"arbitrary":[154],"images,":[155],"these":[156],"cause":[158],"poisoned":[160],"VLM":[161],"generate":[163],"outputs":[164],"semantically":[165],"aligned":[166],"with":[167],"chosen":[169],"caption,":[171],"even":[172],"when":[173],"was":[176],"during":[178],"poisoning.":[179],"Extensive":[180],"experiments":[181],"our":[184],"attack":[185],"achieves":[186],"high":[187],"success":[190],"rates,":[191],"preserves":[192],"clean":[193],"utility,":[195],"remains":[197],"effective":[198],"under":[199],"several":[200],"classical":[201],"defenses.":[203]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-13T00:00:00"}