{"id":"https://openalex.org/W4386998279","doi":"https://doi.org/10.1145/3625290","title":"Hierarchical Distribution-aware Testing of Deep Learning","display_name":"Hierarchical Distribution-aware Testing of Deep Learning","publication_year":2023,"publication_date":"2023-09-24","ids":{"openalex":"https://openalex.org/W4386998279","doi":"https://doi.org/10.1145/3625290"},"language":"en","primary_location":{"id":"doi:10.1145/3625290","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3625290","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3625290","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3625290","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100352892","display_name":"Wei Huang","orcid":"https://orcid.org/0000-0003-1418-6267"},"institutions":[{"id":"https://openalex.org/I4210155350","display_name":"Purple Mountain Laboratories","ror":"https://ror.org/04zcbk583","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210155350"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Wei Huang","raw_affiliation_strings":["Purple Mountain Laboratories, China and University of Liverpool, U.K"],"affiliations":[{"raw_affiliation_string":"Purple Mountain Laboratories, China and University of Liverpool, U.K","institution_ids":["https://openalex.org/I4210155350"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100635681","display_name":"Xingyu Zhao","orcid":"https://orcid.org/0000-0002-3474-349X"},"institutions":[{"id":"https://openalex.org/I39555362","display_name":"University of Warwick","ror":"https://ror.org/01a77tt86","country_code":"GB","type":"education","lineage":["https://openalex.org/I39555362"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Xingyu Zhao","raw_affiliation_strings":["WMG, University of Warwick, U.K"],"affiliations":[{"raw_affiliation_string":"WMG, University of Warwick, U.K","institution_ids":["https://openalex.org/I39555362"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027006867","display_name":"Alec Banks","orcid":"https://orcid.org/0000-0002-1350-0798"},"institutions":[{"id":"https://openalex.org/I1339065421","display_name":"Defence Science and Technology Laboratory","ror":"https://ror.org/04jswqb94","country_code":"GB","type":"government","lineage":["https://openalex.org/I1306956679","https://openalex.org/I1339065421","https://openalex.org/I2802373619"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alec Banks","raw_affiliation_strings":["Defence Science and Technology Laboratory, U.K"],"affiliations":[{"raw_affiliation_string":"Defence Science and Technology Laboratory, U.K","institution_ids":["https://openalex.org/I1339065421"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047316092","display_name":"Victoria Cox","orcid":"https://orcid.org/0000-0003-3852-7855"},"institutions":[{"id":"https://openalex.org/I1339065421","display_name":"Defence Science and Technology Laboratory","ror":"https://ror.org/04jswqb94","country_code":"GB","type":"government","lineage":["https://openalex.org/I1306956679","https://openalex.org/I1339065421","https://openalex.org/I2802373619"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Victoria Cox","raw_affiliation_strings":["Defence Science and Technology Laboratory, U.K"],"affiliations":[{"raw_affiliation_string":"Defence Science and Technology Laboratory, U.K","institution_ids":["https://openalex.org/I1339065421"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020085889","display_name":"Xiaowei Huang","orcid":"https://orcid.org/0000-0001-6267-0366"},"institutions":[{"id":"https://openalex.org/I146655781","display_name":"University of Liverpool","ror":"https://ror.org/04xs57h96","country_code":"GB","type":"education","lineage":["https://openalex.org/I146655781"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Xiaowei Huang","raw_affiliation_strings":["University of Liverpool, U.K"],"affiliations":[{"raw_affiliation_string":"University of Liverpool, U.K","institution_ids":["https://openalex.org/I146655781"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5100352892"],"corresponding_institution_ids":["https://openalex.org/I4210155350"],"apc_list":null,"apc_paid":null,"fwci":1.5525,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.86470026,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"33","issue":"2","first_page":"1","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9778000116348267,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14117","display_name":"Integrated Circuits and Semiconductor Failure Analysis","score":0.9704999923706055,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.808657705783844},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7977559566497803},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.7681957483291626},{"id":"https://openalex.org/keywords/dependability","display_name":"Dependability","score":0.7188301682472229},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5510051846504211},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5319842100143433},{"id":"https://openalex.org/keywords/vulnerability","display_name":"Vulnerability (computing)","score":0.41733109951019287},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3660385310649872},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.15291696786880493},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.07602900266647339}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.808657705783844},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7977559566497803},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.7681957483291626},{"id":"https://openalex.org/C77019957","wikidata":"https://www.wikidata.org/wiki/Q2689057","display_name":"Dependability","level":2,"score":0.7188301682472229},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5510051846504211},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5319842100143433},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.41733109951019287},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3660385310649872},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.15291696786880493},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.07602900266647339},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3625290","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3625290","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3625290","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},{"id":"pmh:oai:wrap.warwick.ac.uk:178933","is_oa":true,"landing_page_url":null,"pdf_url":"https://wrap.warwick.ac.uk/178933/7/WRAP-hierarchical-distribution-aware-testing-deep-learning-Zhao-2023.pdf","source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"}],"best_oa_location":{"id":"doi:10.1145/3625290","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3625290","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3625290","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1361938442","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G1364539167","display_name":"FOUNDATIONS FOR CONTINUOUS ENGINEERING OF TRUSTWORTHY AUTONOMY","funder_award_id":"956123","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G2966875158","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320332972","funder_display_name":"Defence Science and Technology Laboratory"},{"id":"https://openalex.org/G4956428346","display_name":null,"funder_award_id":"Horizon 2020 research and innovatio","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G6661291015","display_name":"EnnCore: End-to-End Conceptual Guarding of Neural Architectures","funder_award_id":"EP/T026995/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8318064016","display_name":null,"funder_award_id":"Horizon","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G849311619","display_name":null,"funder_award_id":"EP/T026995/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320332972","display_name":"Defence Science and Technology Laboratory","ror":"https://ror.org/04jswqb94"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386998279.pdf","grobid_xml":"https://content.openalex.org/works/W4386998279.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W1583837637","https://openalex.org/W1965969360","https://openalex.org/W1968535060","https://openalex.org/W2013807373","https://openalex.org/W2126105956","https://openalex.org/W2133665775","https://openalex.org/W2183341477","https://openalex.org/W2543296129","https://openalex.org/W2560674852","https://openalex.org/W2616028256","https://openalex.org/W2618735189","https://openalex.org/W2804337238","https://openalex.org/W2886368623","https://openalex.org/W2911270308","https://openalex.org/W2912512851","https://openalex.org/W2943498139","https://openalex.org/W2953746822","https://openalex.org/W2962711307","https://openalex.org/W2963913218","https://openalex.org/W2964077693","https://openalex.org/W2968940383","https://openalex.org/W2994987245","https://openalex.org/W3007157104","https://openalex.org/W3011102739","https://openalex.org/W3013948624","https://openalex.org/W3036286896","https://openalex.org/W3042769859","https://openalex.org/W3088864312","https://openalex.org/W3095361491","https://openalex.org/W3103158293","https://openalex.org/W3125205424","https://openalex.org/W3163879210","https://openalex.org/W3188010467","https://openalex.org/W3190746815","https://openalex.org/W4225410698","https://openalex.org/W4241614188","https://openalex.org/W4244364296","https://openalex.org/W4286485305","https://openalex.org/W4289258965","https://openalex.org/W4309618441"],"related_works":["https://openalex.org/W2169628522","https://openalex.org/W4252527915","https://openalex.org/W2409287660","https://openalex.org/W2233357156","https://openalex.org/W1976489385","https://openalex.org/W4256317220","https://openalex.org/W1506225852","https://openalex.org/W2141414364","https://openalex.org/W2376514150","https://openalex.org/W1523265213"],"abstract_inverted_index":{"With":[0],"its":[1,15,117],"growing":[2],"use":[3],"in":[4,48,77,116,212,256],"safety/security-critical":[5],"applications,":[6],"Deep":[7],"Learning":[8],"(DL)":[9],"has":[10,20],"raised":[11],"increasing":[12],"concerns":[13],"regarding":[14],"dependability.":[16],"In":[17,120],"particular,":[18],"DL":[19,42,96,272],"a":[21,90,125,156,197,203,252],"notorious":[22],"problem":[23],"of":[24,67,147,169,175,180,224,258,270],"lacking":[25],"robustness.":[26,177,195],"Input":[27],"added":[28],"with":[29,193],"adversarial":[30,68,148,176],"perturbations,":[31],"i.e.,":[32],"Adversarial":[33],"Examples":[34],"(AEs)":[35],",":[36],"are":[37,58,74,112,153,190],"easily":[38],"mispredicted":[39],"by":[40,155,185],"the":[41,64,71,78,95,100,136,140,144,167,173,186,221,241,267,271],"model.":[43],"Despite":[44],"recent":[45],"efforts":[46],"made":[47],"detecting":[49,109,131,261],"AEs":[50,73,110,132,263],"via":[51],"state-of-the-art":[52],"attack":[53],"and":[54,139,172],"testing":[55,101,128],"methods,":[56],"they":[57],"normally":[59],"input":[60,247],"distribution\u2013agnostic":[61],"and/or":[62],"disregard":[63,245],"perceptual":[65,145,222],"quality":[66,146,223],"perturbations.":[69,149],"Consequently,":[70],"detected":[72,225],"irrelevant":[75],"inputs":[76],"application":[79],"context":[80],"or":[81,249],"noticeably":[82],"unrealistic":[83],"to":[84,89,105,219,240],"humans.":[85],"This":[86],"may":[87],"lead":[88],"limited":[91],"effect":[92],"on":[93,108,166],"improving":[94,266],"model\u2019s":[97],"dependability,":[98],"as":[99],"budget":[102],"is":[103,183,238],"likely":[104],"be":[106],"wasted":[107],"that":[111,133,231,243],"encountered":[113],"very":[114],"rarely":[115],"real-life":[118],"operations.":[119],"this":[121],"article,":[122],"we":[123,161,200],"propose":[124],"new":[126],"robustness":[127,269],"approach":[129,234],"for":[130],"considers":[134],"both":[135],"feature-level":[137,170],"distribution":[138,171,248],"pixel-level":[141],"distribution,":[142,255],"capturing":[143],"The":[150,178],"two":[151,214],"considerations":[152],"encoded":[154],"novel":[157,204],"hierarchical":[158,236],"mechanism.":[159],"First,":[160],"select":[162],"test":[163,181,198,208],"seeds":[164,182],"based":[165],"density":[168],"vulnerability":[174,179],"indicated":[184],"auxiliary":[187],"information,":[188],"which":[189,213],"highly":[191],"correlated":[192],"local":[194,207],"Given":[196],"seed,":[199],"then":[201],"develop":[202],"genetic":[205],"algorithm\u2013based":[206],"case":[209],"generation":[210],"method,":[211],"fitness":[215],"functions":[216],"work":[217],"alternatively":[218],"control":[220],"AEs.":[226],"Finally,":[227],"extensive":[228],"experiments":[229],"confirm":[230],"our":[232],"holistic":[233],"considering":[235],"distributions":[237],"superior":[239],"state-of-the-arts":[242],"either":[244],"any":[246],"only":[250,260],"consider":[251],"single":[253],"(non-hierarchical)":[254],"terms":[257],"not":[259],"imperceptible":[262],"but":[264],"also":[265],"overall":[268],"model":[273],"under":[274],"testing.":[275]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":4}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}