{"id":"https://openalex.org/W4387074881","doi":"https://doi.org/10.48550/arxiv.2309.13245","title":"RBFormer: Improve Adversarial Robustness of Transformer by Robust Bias","display_name":"RBFormer: Improve Adversarial Robustness of Transformer by Robust Bias","publication_year":2023,"publication_date":"2023-09-23","ids":{"openalex":"https://openalex.org/W4387074881","doi":"https://doi.org/10.48550/arxiv.2309.13245"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2309.13245","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.13245","pdf_url":"https://arxiv.org/pdf/2309.13245","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2309.13245","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110531341","display_name":"Hao Cheng","orcid":"https://orcid.org/0000-0002-3246-6636","observed_orcids":["https://orcid.org/0000-0002-3246-6636"]},"institutions":[{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cheng, Hao","raw_affiliation_strings":["The Hong Kong University of Sci- ence and Technology (Guangzhou) , Guangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Sci- ence and Technology (Guangzhou) , Guangzhou , China","institution_ids":["https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073311852","display_name":"Jinhao Duan","orcid":"https://orcid.org/0000-0001-7045-9376","observed_orcids":["https://orcid.org/0000-0001-7045-9376"]},"institutions":[{"id":"https://openalex.org/I72816309","display_name":"Drexel University","ror":"https://ror.org/04bdffz58","country_code":"US","type":"education","lineage":["https://openalex.org/I72816309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Duan, Jinhao","raw_affiliation_strings":["Drexel University , Philadelphia , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Drexel University , Philadelphia , USA","institution_ids":["https://openalex.org/I72816309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025520267","display_name":"Hui Li","orcid":"https://orcid.org/0000-0002-9222-6220","observed_orcids":["https://orcid.org/0000-0002-9222-6220"]},"institutions":[{"id":"https://openalex.org/I4210155230","display_name":"Samsung (China)","ror":"https://ror.org/04yt00889","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250650973","https://openalex.org/I4210155230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Hui","raw_affiliation_strings":["Samsung Research and Development Institute China Xi'an , Xi'an , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung Research and Development Institute China Xi'an , Xi'an , China","institution_ids":["https://openalex.org/I4210155230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018244506","display_name":"Lyutianyang Zhang","orcid":"https://orcid.org/0000-0001-9789-946X","observed_orcids":["https://orcid.org/0000-0001-9789-946X"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Lyutianyang","raw_affiliation_strings":["University of Washington , Seattle , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington , Seattle , USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015422529","display_name":"Jiahang Cao","orcid":"https://orcid.org/0000-0003-4338-4414","observed_orcids":["https://orcid.org/0000-0003-4338-4414"]},"institutions":[{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cao, Jiahang","raw_affiliation_strings":["The Hong Kong University of Sci- ence and Technology (Guangzhou) , Guangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Sci- ence and Technology (Guangzhou) , Guangzhou , China","institution_ids":["https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100338623","display_name":"Ping Wang","orcid":"https://orcid.org/0000-0002-0379-9183","observed_orcids":["https://orcid.org/0000-0002-0379-9183","https://orcid.org/0000-0003-0033-4150"]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Ping","raw_affiliation_strings":["Xi'an Jiaotong University , Xi'an , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University , Xi'an , China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060905917","display_name":"Jize Zhang","orcid":"https://orcid.org/0000-0002-0553-3979","observed_orcids":["https://orcid.org/0000-0002-0553-3979"]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhang, Jize","raw_affiliation_strings":["The Hong Kong University of Science and Technology , Hong Kong SAR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology , Hong Kong SAR","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102775611","display_name":"Kaidi Xu","orcid":"https://orcid.org/0000-0002-0719-3912","observed_orcids":["https://orcid.org/0000-0002-0719-3912","https://orcid.org/0000-0003-4437-0671"]},"institutions":[{"id":"https://openalex.org/I72816309","display_name":"Drexel University","ror":"https://ror.org/04bdffz58","country_code":"US","type":"education","lineage":["https://openalex.org/I72816309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Kaidi","raw_affiliation_strings":["Drexel University , Philadelphia , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Drexel University , Philadelphia , USA","institution_ids":["https://openalex.org/I72816309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100616654","display_name":"Renjing Xu","orcid":"https://orcid.org/0000-0002-0792-8974","observed_orcids":["https://orcid.org/0000-0002-0792-8974"]},"institutions":[{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu, Renjing","raw_affiliation_strings":["The Hong Kong University of Sci- ence and Technology (Guangzhou) , Guangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Sci- ence and Technology (Guangzhou) , Guangzhou , China","institution_ids":["https://openalex.org/I4405261251"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-robustness","display_name":"adversarial robustness","score":0.9800000190734863},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.4169999957084656},{"id":"https://openalex.org/keywords/vision-transformer","display_name":"vision transformer","score":0.7900000214576721},{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.6150000095367432},{"id":"https://openalex.org/keywords/structural-design","display_name":"structural design","score":0.5460000038146973},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.7519999742507935}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6803904175758362},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.6593875885009766},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6368314027786255},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.571985125541687},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4432445466518402},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.369911253452301},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.20596981048583984},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.18464183807373047},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.07827591896057129},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2309.13245","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.13245","pdf_url":"https://arxiv.org/pdf/2309.13245","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-138283","doi":null,"is_oa":false,"landing_page_url":"http://lbdiscover.ust.hk/uresolver?url_ver=Z39.88-2004&rft_val_fmt=info:ofi/fmt:kev:mtx:journal&rfr_id=info:sid/HKUST:SPI&rft.genre=article&rft.issn=&rft.volume=&rft.issue=&rft.date=2023&rft.spage=&rft.aulast=Cheng&rft.aufirst=Hao&rft.atitle=RBFormer%3A+Improve+Adversarial+Robustness+of+Transformer+by+Robust+Bias&rft.title=","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"},{"id":"doi:10.48550/arxiv.2309.13245","doi":"https://doi.org/10.48550/arxiv.2309.13245","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2309.13245","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2309.13245","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.13245","pdf_url":"https://arxiv.org/pdf/2309.13245","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387074881.pdf","grobid_xml":"https://content.openalex.org/works/W4387074881.grobid-xml"},"referenced_works_count":6,"referenced_works":["https://openalex.org/W1673923490","https://openalex.org/W1686810756","https://openalex.org/W3037742886","https://openalex.org/W4283809036","https://openalex.org/W4308245819","https://openalex.org/W4389666313"],"related_works":["https://openalex.org/W3142085127","https://openalex.org/W4404534699","https://openalex.org/W3214243459","https://openalex.org/W7140090481","https://openalex.org/W4308245819","https://openalex.org/W4415800563","https://openalex.org/W2952085755","https://openalex.org/W3143373604","https://openalex.org/W2912087091","https://openalex.org/W3171408944"],"abstract_inverted_index":{"Recently,":[0],"there":[1],"has":[2,55,75],"been":[3,77],"a":[4,35,52,57,114,142,146,166,177],"surge":[5],"of":[6,65,93,129,136,168,184],"interest":[7],"and":[8,18,59,186,194],"attention":[9],"in":[10,79],"Transformer-based":[11,30,80,152],"structures,":[12,28],"such":[13,121],"as":[14,51],"Vision":[15,19],"Transformer":[16],"(ViT)":[17],"Multilayer":[20],"Perceptron":[21],"(VMLP).":[22],"Compared":[23],"with":[24],"the":[25,29,63,90,94,107,126,130,134,173],"previous":[26],"convolution-based":[27,67],"structure":[31,95,116,131,148],"under":[32,40],"investigation":[33],"showcases":[34],"comparable":[36],"or":[37],"superior":[38],"performance":[39,64],"its":[41],"distinctive":[42],"attention-based":[43],"input":[44],"token":[45],"mixer":[46],"strategy.":[47],"Introducing":[48],"adversarial":[49,73,103,127],"examples":[50],"robustness":[53,92,110,128],"consideration":[54],"had":[56],"profound":[58],"detrimental":[60],"impact":[61],"on":[62,88,192],"well-established":[66],"structures.":[68,81,164],"This":[69],"inherent":[70],"vulnerability":[71],"to":[72,109,119,160],"attacks":[74],"also":[76],"demonstrated":[78],"In":[82],"this":[83],"paper,":[84],"our":[85],"emphasis":[86],"lies":[87],"investigating":[89],"intrinsic":[91],"rather":[96],"than":[97],"introducing":[98],"novel":[99,147],"defense":[100],"measures":[101],"against":[102],"attacks.":[104],"To":[105],"address":[106],"susceptibility":[108],"issues,":[111],"we":[112,124,144],"employ":[113],"rational":[115],"design":[117],"approach":[118],"mitigate":[120],"vulnerabilities.":[122],"Specifically,":[123],"enhance":[125],"by":[132,176],"increasing":[133],"proportion":[135],"high-frequency":[137],"structural":[138],"robust":[139,157],"biases.":[140],"As":[141],"result,":[143],"introduce":[145],"called":[149],"Robust":[150],"Bias":[151],"Structure":[153],"(RBFormer)":[154],"that":[155],"shows":[156],"superiority":[158],"compared":[159],"several":[161],"existing":[162],"baseline":[163],"Through":[165],"series":[167],"extensive":[169],"experiments,":[170],"RBFormer":[171],"outperforms":[172],"original":[174],"structures":[175],"significant":[178],"margin,":[179],"achieving":[180],"an":[181],"impressive":[182],"improvement":[183],"+16.12%":[185],"+5.04%":[187],"across":[188],"different":[189],"evaluation":[190],"criteria":[191],"CIFAR-10":[193],"ImageNet-1k,":[195],"respectively.":[196]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-09-27T00:00:00"}