{"id":"https://openalex.org/W4206813776","doi":"https://doi.org/10.1109/ase51524.2021.9678540","title":"DEVIATE: A Deep Learning Variance Testing Framework","display_name":"DEVIATE: A Deep Learning Variance Testing Framework","publication_year":2021,"publication_date":"2021-11-01","ids":{"openalex":"https://openalex.org/W4206813776","doi":"https://doi.org/10.1109/ase51524.2021.9678540"},"language":"en","primary_location":{"id":"doi:10.1109/ase51524.2021.9678540","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ase51524.2021.9678540","pdf_url":null,"source":{"id":"https://openalex.org/S4363608212","display_name":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002648627","display_name":"Hung Viet Pham","orcid":"https://orcid.org/0000-0002-1769-8527"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Hung Viet Pham","raw_affiliation_strings":["University of Waterloo"],"affiliations":[{"raw_affiliation_string":"University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100770931","display_name":"Mijung Kim","orcid":"https://orcid.org/0000-0003-3995-5384"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mijung Kim","raw_affiliation_strings":["Purdue University"],"affiliations":[{"raw_affiliation_string":"Purdue University","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115883174","display_name":"Lin Tan","orcid":"https://orcid.org/0000-0002-6690-8332"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lin Tan","raw_affiliation_strings":["Purdue University"],"affiliations":[{"raw_affiliation_string":"Purdue University","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104247245","display_name":"Yaoliang Yu","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Yaoliang Yu","raw_affiliation_strings":["University of Waterloo"],"affiliations":[{"raw_affiliation_string":"University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101612059","display_name":"Nachiappan Nagappan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Nachiappan Nagappan","raw_affiliation_strings":["Microsoft Research"],"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I4210164937"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5002648627"],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":0.7612,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.75522278,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1286","last_page":"1290"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9690999984741211,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.8154821395874023},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7979731559753418},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.7872384786605835},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.6370687484741211},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5448799133300781},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5373021960258484},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5177847146987915},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5077815651893616},{"id":"https://openalex.org/keywords/nondeterministic-algorithm","display_name":"Nondeterministic algorithm","score":0.5045818090438843},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3459867238998413},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.32344043254852295},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.26894912123680115},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12181288003921509},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.08862915635108948}],"concepts":[{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.8154821395874023},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7979731559753418},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.7872384786605835},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.6370687484741211},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5448799133300781},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5373021960258484},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5177847146987915},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5077815651893616},{"id":"https://openalex.org/C176181172","wikidata":"https://www.wikidata.org/wiki/Q3490301","display_name":"Nondeterministic algorithm","level":2,"score":0.5045818090438843},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3459867238998413},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.32344043254852295},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.26894912123680115},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12181288003921509},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.08862915635108948},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/ase51524.2021.9678540","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ase51524.2021.9678540","pdf_url":null,"source":{"id":"https://openalex.org/S4363608212","display_name":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","raw_type":"proceedings-article"},{"id":"pmh:oai:scholarworks.unist.ac.kr:201301/53975","is_oa":false,"landing_page_url":"https://conf.researchr.org/details/ase-2021/ase-2021-tool-demonstrations/6/DEVIATE-A-Deep-Learning-Variance-Testing-Framework","pdf_url":null,"source":{"id":"https://openalex.org/S4306401118","display_name":"Scholarworks@UNIST (Ulsan National Institute of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I48566637","host_organization_name":"Ulsan National Institute of Science and Technology","host_organization_lineage":["https://openalex.org/I48566637"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"CONFERENCE"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4000000059604645,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1945616565","https://openalex.org/W2130942839","https://openalex.org/W2141125852","https://openalex.org/W2160815625","https://openalex.org/W2342840547","https://openalex.org/W2565186948","https://openalex.org/W2592929672","https://openalex.org/W2734556049","https://openalex.org/W2777646139","https://openalex.org/W2796394805","https://openalex.org/W2801177523","https://openalex.org/W2883457665","https://openalex.org/W2963674387","https://openalex.org/W2964330541","https://openalex.org/W3000315285","https://openalex.org/W4240402692","https://openalex.org/W4289543668","https://openalex.org/W4293846201","https://openalex.org/W4294583674","https://openalex.org/W6640425456","https://openalex.org/W6679436768","https://openalex.org/W6704559304","https://openalex.org/W6739868092","https://openalex.org/W6746593864","https://openalex.org/W6752891299","https://openalex.org/W6754110879"],"related_works":["https://openalex.org/W4254851101","https://openalex.org/W3171007296","https://openalex.org/W22115721","https://openalex.org/W2321234655","https://openalex.org/W2065444835","https://openalex.org/W4394550905","https://openalex.org/W2470062578","https://openalex.org/W2502115930","https://openalex.org/W2952773340","https://openalex.org/W2981861370"],"abstract_inverted_index":{"Deep":[0],"learning":[1],"(DL)":[2],"training":[3,85,111,168,183,195],"is":[4,72],"nondeterministic":[5],"and":[6,43,62,65,70,74,95,103,113,121,130,139,206],"such":[7,39],"nondeterminism":[8],"was":[9],"shown":[10],"to":[11,19,54,149,177,201,204],"cause":[12],"significant":[13],"variance":[14,22,57,86],"of":[15,27,30,58,87,100,136,155,190],"model":[16,90],"accuracy":[17,102,120,191],"(up":[18],"10.8%).":[20],"Such":[21],"may":[23],"affect":[24],"the":[25,28,56,59,109,153,172,182],"validity":[26],"comparison":[29],"newly":[31],"proposed":[32,60],"DL":[33,41,67,84,89,110],"techniques":[34],"with":[35,51,91,133,143,161,197],"baselines.":[36,63],"To":[37],"ensure":[38],"validity,":[40],"researchers":[42],"practitioners":[44],"must":[45],"replicate":[46],"their":[47],"experiments":[48],"multiple":[49],"times":[50],"identical":[52,194],"settings":[53],"quantify":[55],"approaches":[61],"Replicating":[64],"measuring":[66],"variances":[68],"reliably":[69],"efficiently":[71],"challenging":[73],"understudied.We":[75],"propose":[76],"a":[77,88,134,187],"ready-to-deploy":[78],"framework":[79],"DEVIATE":[80,106,125,156,185],"that":[81,170],"(1)":[82],"measures":[83,186],"minimal":[92],"manual":[93],"efforts,":[94],"(2)":[96],"provides":[97,131],"statistical":[98,128,137],"tests":[99,129],"both":[101],"variance.":[104],"Specifically,":[105,164],"automatically":[107],"analyzes":[108],"code":[112],"extracts":[114],"monitored":[115],"important":[116],"metrics":[117],"(such":[118],"as":[119,181],"loss).":[122],"In":[123],"addition,":[124],"performs":[126],"popular":[127],"users":[132],"report":[135],"p-values":[138],"effect":[140],"sizes":[141],"along":[142],"various":[144],"confidence":[145],"levels":[146],"when":[147],"comparing":[148],"selected":[150],"baselines.We":[151],"demonstrate":[152],"effectiveness":[154],"by":[157],"performing":[158],"case":[159],"studies":[160],"adversarial":[162,167,179],"training.":[163],"for":[165],"an":[166],"process":[169],"uses":[171],"Fast":[173],"Gradient":[174],"Signed":[175],"Method":[176],"generate":[178],"examples":[180],"data,":[184],"max":[188],"difference":[189],"among":[192],"8":[193],"runs":[196],"fixed":[198],"random":[199],"seeds":[200],"be":[202],"up":[203],"5.1%.Tool":[205],"demo":[207],"links:":[208],"https://github.com/lin-tan/DEVIATE":[209]},"counts_by_year":[{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}