{"id":"https://openalex.org/W4410809650","doi":"https://doi.org/10.1109/tpami.2025.3574432","title":"BlackboxBench: A Comprehensive Benchmark of Black-Box Adversarial Attacks","display_name":"BlackboxBench: A Comprehensive Benchmark of Black-Box Adversarial Attacks","publication_year":2025,"publication_date":"2025-05-28","ids":{"openalex":"https://openalex.org/W4410809650","doi":"https://doi.org/10.1109/tpami.2025.3574432","pmid":"https://pubmed.ncbi.nlm.nih.gov/40434861"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3574432","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3574432","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028139500","display_name":"Meixi Zheng","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Meixi Zheng","raw_affiliation_strings":["School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China"],"affiliations":[{"raw_affiliation_string":"School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065664599","display_name":"Xuanchen Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuanchen Yan","raw_affiliation_strings":["School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China"],"affiliations":[{"raw_affiliation_string":"School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101190196","display_name":"Zihao Zhu","orcid":"https://orcid.org/0009-0004-1063-387X"},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zihao Zhu","raw_affiliation_strings":["School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China"],"affiliations":[{"raw_affiliation_string":"School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002810045","display_name":"Hongrui Chen","orcid":"https://orcid.org/0009-0009-8517-4379"},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongrui Chen","raw_affiliation_strings":["School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China"],"affiliations":[{"raw_affiliation_string":"School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068027800","display_name":"Baoyuan Wu","orcid":"https://orcid.org/0000-0003-2183-5990"},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Baoyuan Wu","raw_affiliation_strings":["School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China"],"affiliations":[{"raw_affiliation_string":"School of Data Science, Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"School of Data Science, The Chinese University of Hong Kong, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I4210116924"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5028139500"],"corresponding_institution_ids":["https://openalex.org/I4210116924"],"apc_list":null,"apc_paid":null,"fwci":21.2115,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.99226417,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"47","issue":"9","first_page":"7867","last_page":"7885"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12122","display_name":"Physical Unclonable Functions (PUFs) and Hardware Security","score":0.961899995803833,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9564999938011169,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.7742481827735901},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7347439527511597},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6859527230262756},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6591637134552002},{"id":"https://openalex.org/keywords/black-box","display_name":"Black box","score":0.6068778038024902},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.40663692355155945},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36209559440612793}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.7742481827735901},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7347439527511597},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6859527230262756},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6591637134552002},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.6068778038024902},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.40663692355155945},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36209559440612793},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2025.3574432","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3574432","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40434861","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40434861","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6479059842","display_name":null,"funder_award_id":"2024B1515020095","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320337111","display_name":"Basic and Applied Basic Research Foundation of Guangdong Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":87,"referenced_works":["https://openalex.org/W2117539524","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2774644650","https://openalex.org/W2902543210","https://openalex.org/W2942039787","https://openalex.org/W2950808778","https://openalex.org/W2962847335","https://openalex.org/W2963542245","https://openalex.org/W2969542116","https://openalex.org/W2972986629","https://openalex.org/W2984699060","https://openalex.org/W2997583194","https://openalex.org/W3009153202","https://openalex.org/W3015625436","https://openalex.org/W3034214559","https://openalex.org/W3034619610","https://openalex.org/W3034892461","https://openalex.org/W3035172095","https://openalex.org/W3043119335","https://openalex.org/W3080297477","https://openalex.org/W3094502228","https://openalex.org/W3105976275","https://openalex.org/W3107235539","https://openalex.org/W3109877713","https://openalex.org/W3127807678","https://openalex.org/W3138516171","https://openalex.org/W3143373604","https://openalex.org/W3171288285","https://openalex.org/W3178946670","https://openalex.org/W3186991201","https://openalex.org/W3209714106","https://openalex.org/W3212790025","https://openalex.org/W4224950673","https://openalex.org/W4225859735","https://openalex.org/W4293846201","https://openalex.org/W4304701424","https://openalex.org/W4312245536","https://openalex.org/W4312443924","https://openalex.org/W4312513141","https://openalex.org/W4312697129","https://openalex.org/W4312790346","https://openalex.org/W4312809370","https://openalex.org/W4316660635","https://openalex.org/W4381232913","https://openalex.org/W4389633739","https://openalex.org/W4390871946","https://openalex.org/W4390872540","https://openalex.org/W4390873445","https://openalex.org/W4411244506","https://openalex.org/W6637373629","https://openalex.org/W6714069269","https://openalex.org/W6731927902","https://openalex.org/W6746608116","https://openalex.org/W6750404860","https://openalex.org/W6752654261","https://openalex.org/W6752931940","https://openalex.org/W6752985256","https://openalex.org/W6753044988","https://openalex.org/W6759158001","https://openalex.org/W6759204839","https://openalex.org/W6759580348","https://openalex.org/W6762681652","https://openalex.org/W6767666165","https://openalex.org/W6768366551","https://openalex.org/W6771543752","https://openalex.org/W6772823604","https://openalex.org/W6773713311","https://openalex.org/W6777386883","https://openalex.org/W6783920880","https://openalex.org/W6784493056","https://openalex.org/W6785111489","https://openalex.org/W6787972765","https://openalex.org/W6790824169","https://openalex.org/W6799114475","https://openalex.org/W6803004494","https://openalex.org/W6803084059","https://openalex.org/W6844413036","https://openalex.org/W6845892164","https://openalex.org/W6850063601","https://openalex.org/W6850389943","https://openalex.org/W6850740368","https://openalex.org/W6853649152","https://openalex.org/W6857595869","https://openalex.org/W6857601782","https://openalex.org/W6864274290","https://openalex.org/W6929167720"],"related_works":["https://openalex.org/W2502115930","https://openalex.org/W2482350142","https://openalex.org/W4246396837","https://openalex.org/W3126451824","https://openalex.org/W1561927205","https://openalex.org/W3191453585","https://openalex.org/W4297672492","https://openalex.org/W4310988119","https://openalex.org/W4385437088","https://openalex.org/W3009622996"],"abstract_inverted_index":{"Adversarial":[0],"examples":[1],"are":[2,36],"well-known":[3],"tools":[4],"to":[5,38,46,73,171],"evaluate":[6,150],"the":[7,27,31,39,47,60,74,91,151],"vulnerability":[8],"of":[9,16,98,109,120,168],"deep":[10],"neural":[11],"networks":[12],"(DNNs).":[13],"Although":[14],"lots":[15],"adversarial":[17,43,122],"attack":[18,64,139,144],"algorithms":[19,140,153],"have":[20],"been":[21,52],"developed,":[22],"it's":[23],"still":[24],"challenging":[25],"in":[26,63,70,89,174],"practical":[28,48],"scenario":[29],"that":[30],"model's":[32],"parameters":[33],"and":[34,67,82,96,133,141,165,179],"architectures":[35,158],"inaccessible":[37],"attacker/evaluator,":[40],"i.e.,":[41],"black-box":[42,121],"attacks.":[44],"Due":[45],"importance,":[49],"there":[50,78],"has":[51],"rapid":[53],"progress":[54],"from":[55],"recent":[56],"algorithms,":[57,86],"reflected":[58],"by":[59],"quick":[61,68],"increase":[62],"success":[65],"rate":[66],"decrease":[69],"query":[71],"numbers":[72],"target":[75],"model.":[76],"However,":[77],"lacks":[79],"thorough":[80,177],"evaluations":[81],"comparisons":[83],"among":[84],"these":[85],"causing":[87],"difficulties":[88],"tracking":[90],"real":[92],"progress,":[93],"analyzing":[94],"advantages":[95],"disadvantages":[97],"different":[99],"technical":[100],"routes,":[101],"as":[102,104,182],"well":[103,183],"designing":[105],"future":[106],"development":[107],"roadmap":[108],"this":[110],"field.":[111],"Thus,":[112],"we":[113,149],"aim":[114],"at":[115],"building":[116],"a":[117,130,166],"comprehensive":[118,147],"benchmark":[119],"attacks,":[123],"called":[124],"BlackboxBench.":[125],"It":[126],"mainly":[127],"provides:":[128],"1)":[129],"unified,":[131],"extensible":[132],"modular-based":[134],"codebase,":[135],"implementing":[136],"29":[137],"query-based":[138],"30":[142],"transfer-based":[143],"algorithms;":[145],"2)":[146],"evaluations:":[148],"implemented":[152],"against":[154],"several":[155],"mainstreaming":[156],"model":[157],"on":[159],"2":[160],"widely":[161],"used":[162],"datasets":[163],"(CIFAR-10":[164],"subset":[167],"ImageNet),":[169],"leading":[170],"14,950":[172],"evaluations$^{1}$1":[173],"total;":[175],"3)":[176],"analysis":[178],"new":[180],"insights,":[181],"analytical":[184],"tools.":[185]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":5}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}