{"id":"https://openalex.org/W7126258834","doi":"https://doi.org/10.1109/tpami.2026.3659598","title":"NAP-Tuning: Neural Augmented Prompt Tuning for Adversarially Robust Vision-Language Models","display_name":"NAP-Tuning: Neural Augmented Prompt Tuning for Adversarially Robust Vision-Language Models","publication_year":2026,"publication_date":"2026-01-30","ids":{"openalex":"https://openalex.org/W7126258834","doi":"https://doi.org/10.1109/tpami.2026.3659598","pmid":"https://pubmed.ncbi.nlm.nih.gov/41615981"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2026.3659598","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3659598","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5124361068","display_name":"Jiaming Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaming Zhang","raw_affiliation_strings":["School of Computer Science and Technology, Beijing Jiaotong University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0991-7109","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124407879","display_name":"Xin Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Wang","raw_affiliation_strings":["Shanghai Key Lab of Intell. Info. Processing, School of CS, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-9531-6662","affiliations":[{"raw_affiliation_string":"Shanghai Key Lab of Intell. Info. Processing, School of CS, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xingjun Ma","orcid":"https://orcid.org/0000-0003-2099-4973"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xingjun Ma","raw_affiliation_strings":["Shanghai Key Lab of Intell. Info. Processing, School of CS, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-2099-4973","affiliations":[{"raw_affiliation_string":"Shanghai Key Lab of Intell. Info. Processing, School of CS, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110706900","display_name":"Lingyu Qiu","orcid":"https://orcid.org/0009-0001-0571-8156"},"institutions":[{"id":"https://openalex.org/I71267560","display_name":"University of Naples Federico II","ror":"https://ror.org/05290cv24","country_code":"IT","type":"education","lineage":["https://openalex.org/I71267560"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Lingyu Qiu","raw_affiliation_strings":["Department of Mathematics and Applications “R. Caccioppoli”, University of Naples Federico II, Via Cintia, Monte S. Angelo, Naples, Italy"],"raw_orcid":"https://orcid.org/0009-0001-0571-8156","affiliations":[{"raw_affiliation_string":"Department of Mathematics and Applications “R. Caccioppoli”, University of Naples Federico II, Via Cintia, Monte S. Angelo, Naples, Italy","institution_ids":["https://openalex.org/I71267560"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124410451","display_name":"Yu-Gang Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu-Gang Jiang","raw_affiliation_strings":["Shanghai Key Lab of Intell. Info. Processing, School of CS, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-1907-8567","affiliations":[{"raw_affiliation_string":"Shanghai Key Lab of Intell. Info. Processing, School of CS, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5124361438","display_name":"Jitao Sang","orcid":null},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jitao Sang","raw_affiliation_strings":["State Key Laboratory of Advanced Rail Autonomous Operation, Beijing Jiaotong University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-0699-3205","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Advanced Rail Autonomous Operation, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.11002958,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"48","issue":"6","first_page":"6615","last_page":"6627"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9447000026702881,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9447000026702881,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.013100000098347664,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.00930000003427267,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.8797000050544739},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6995000243186951},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.6801000237464905},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5127000212669373},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.42260000109672546},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4189999997615814},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.4140999913215637},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.3434000015258789}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.8797000050544739},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7764999866485596},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6995000243186951},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.6801000237464905},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6674000024795532},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5465999841690063},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5127000212669373},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.42260000109672546},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4189999997615814},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4140999913215637},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.3434000015258789},{"id":"https://openalex.org/C75294576","wikidata":"https://www.wikidata.org/wiki/Q5165192","display_name":"Contextual image classification","level":3,"score":0.3384999930858612},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.33649998903274536},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.3327000141143799},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.3156000077724457},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3140999972820282},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.26809999346733093},{"id":"https://openalex.org/C2778827112","wikidata":"https://www.wikidata.org/wiki/Q22245680","display_name":"Feature engineering","level":3,"score":0.26100000739097595},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C83665646","wikidata":"https://www.wikidata.org/wiki/Q42139305","display_name":"Feature vector","level":2,"score":0.2587999999523163},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25619998574256897}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2026.3659598","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2026.3659598","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:41615981","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41615981","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3805773779","display_name":null,"funder_award_id":"62576030","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4961660815","display_name":null,"funder_award_id":"62572040","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W12634471","https://openalex.org/W1977295328","https://openalex.org/W2017814585","https://openalex.org/W2047643928","https://openalex.org/W2117539524","https://openalex.org/W2138011018","https://openalex.org/W2155904486","https://openalex.org/W2533598788","https://openalex.org/W2774644650","https://openalex.org/W2962847335","https://openalex.org/W2964194231","https://openalex.org/W3046768359","https://openalex.org/W3171288285","https://openalex.org/W3174770825","https://openalex.org/W3198377975","https://openalex.org/W4285247752","https://openalex.org/W4312310776","https://openalex.org/W4382449549","https://openalex.org/W4386071547","https://openalex.org/W4390873848","https://openalex.org/W4402727764","https://openalex.org/W4402783842","https://openalex.org/W4404654839","https://openalex.org/W4413144264","https://openalex.org/W4413146434","https://openalex.org/W4413156116","https://openalex.org/W4415799141"],"related_works":[],"abstract_inverted_index":{"Vision-Language":[0],"Models":[1],"(VLMs)":[2],"such":[3],"as":[4],"CLIP":[5],"have":[6],"demonstrated":[7],"remarkable":[8],"capabilities":[9],"in":[10,33,60,141,219],"understanding":[11],"relationships":[12],"between":[13],"visual":[14],"and":[15,94,96,153,160,176,200,239],"textual":[16],"data":[17],"through":[18,114],"joint":[19],"embedding":[20,244],"spaces.":[21,245],"Despite":[22],"their":[23],"effectiveness,":[24],"these":[25],"models":[26],"remain":[27],"vulnerable":[28],"to":[29,56,131,156,230],"adversarial":[30,58,223,241],"attacks,":[31],"particularly":[32],"the":[34,73,142,151,186,190,213],"image":[35],"modality,":[36],"posing":[37],"significant":[38,69,85,183],"security":[39],"concerns.":[40],"Building":[41],"upon":[42],"our":[43,115,180],"previous":[44],"work":[45,211],"on":[46,198,202],"Adversarial":[47,79],"Prompt":[48,80],"Tuning":[49,81],"(AdvPT),":[50],"which":[51],"introduced":[52],"learnable":[53],"text":[54],"prompts":[55],"enhance":[57],"robustness":[59],"VLMs":[61],"without":[62],"extensive":[63],"parameter":[64],"training,":[65],"we":[66],"present":[67],"a":[68,84,90,106],"extension":[70],"by":[71,124,196],"introducing":[72],"Neural":[74,116],"Augmentor":[75,117],"framework":[76,104,120],"for":[77,110,222],"Multi-modal":[78],"(NAP-Tuning).":[82],"As":[83],"extension,":[86],"NAP-Tuning":[87,168],"first":[88],"establishes":[89],"comprehensive":[91],"multi-modal":[92,152],"(text":[93],"visual)":[95],"multi-layer":[97,154],"prompting":[98],"framework.":[99],"The":[100],"core":[101],"of":[102,215],"this":[103],"is":[105,148],"targeted":[107],"structural":[108,146],"augmentation":[109],"feature-level":[111,217],"purification,":[112],"implemented":[113],"approach.":[118],"This":[119,145,210],"implements":[121],"feature":[122,143,162],"purification":[123],"incorporating":[125],"TokenRefiners-lightweight":[126],"neural":[127],"modules":[128],"that":[129,167,236],"learn":[130],"reconstruct":[132],"purified":[133],"features":[134],"via":[135],"residual":[136],"connections-to":[137],"directly":[138],"address":[139],"distortions":[140],"space.":[144],"intervention":[147,218],"what":[149],"enables":[150],"system":[155],"effectively":[157],"perform":[158],"modality-specific":[159],"layer-specific":[161],"rectification.":[163],"Comprehensive":[164],"experiments":[165],"demonstrate":[166],"significantly":[169],"outperforms":[170],"existing":[171],"methods":[172],"across":[173,243],"various":[174],"datasets":[175],"attack":[177],"types.":[178],"Notably,":[179],"approach":[181],"shows":[182],"improvements":[184],"over":[185],"strongest":[187],"baselines":[188],"under":[189],"challenging":[191],"AutoAttack":[192],"benchmark,":[193],"outperforming":[194],"them":[195],"32.3%":[197],"ViT-B16":[199],"31.3%":[201],"ViT-B32":[203],"architectures":[204],"while":[205],"maintaining":[206],"competitive":[207],"clean":[208],"accuracy.":[209],"highlights":[212],"efficacy":[214],"internal":[216],"prompt":[220],"tuning":[221],"robustness,":[224],"moving":[225],"beyond":[226],"input-side":[227],"alignment":[228],"approaches":[229],"create":[231],"an":[232],"adaptive":[233],"defense":[234],"mechanism":[235],"can":[237],"identify":[238],"rectify":[240],"perturbations":[242]},"counts_by_year":[],"updated_date":"2026-05-09T06:09:20.037420","created_date":"2026-02-01T00:00:00"}