{"id":"https://openalex.org/W3041012898","doi":"https://doi.org/10.1145/3394112","title":"Practical Accuracy Estimation for Efficient Deep Neural Network Testing","display_name":"Practical Accuracy Estimation for Efficient Deep Neural Network Testing","publication_year":2020,"publication_date":"2020-07-07","ids":{"openalex":"https://openalex.org/W3041012898","doi":"https://doi.org/10.1145/3394112","mag":"3041012898"},"language":"en","primary_location":{"id":"doi:10.1145/3394112","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394112","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100365555","display_name":"Junjie Chen","orcid":"https://orcid.org/0000-0003-3056-9962"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Junjie Chen","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103931115","display_name":"Zhuo Wu","orcid":"https://orcid.org/0009-0006-0165-8746"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhuo Wu","raw_affiliation_strings":["Tianjin International Engineering Institute, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"Tianjin International Engineering Institute, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100707447","display_name":"Zan Wang","orcid":"https://orcid.org/0000-0001-6173-8170"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zan Wang","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047258377","display_name":"Hanmo You","orcid":"https://orcid.org/0000-0003-2870-7801"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanmo You","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043546718","display_name":"Lingming Zhang","orcid":"https://orcid.org/0000-0001-5175-2702"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lingming Zhang","raw_affiliation_strings":["Department of Computer Science, The University of Texas at Dallas, Richardson, TX, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, The University of Texas at Dallas, Richardson, TX, USA","institution_ids":["https://openalex.org/I162577319"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077222235","display_name":"Ming Yan","orcid":"https://orcid.org/0000-0001-9757-7794"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ming Yan","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100365555"],"corresponding_institution_ids":["https://openalex.org/I162868743"],"apc_list":null,"apc_paid":null,"fwci":7.5349,"has_fulltext":false,"cited_by_count":91,"citation_normalized_percentile":{"value":0.97768631,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"29","issue":"4","first_page":"1","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.823441743850708},{"id":"https://openalex.org/keywords/pace","display_name":"Pace","score":0.7418486475944519},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6603377461433411},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.623502254486084},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6080211997032166},{"id":"https://openalex.org/keywords/test-set","display_name":"Test set","score":0.5692426562309265},{"id":"https://openalex.org/keywords/random-testing","display_name":"Random testing","score":0.5425174832344055},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5160545706748962},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5018348693847656},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4802258610725403},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36421093344688416},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.27530378103256226},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.2459251582622528}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.823441743850708},{"id":"https://openalex.org/C2777526511","wikidata":"https://www.wikidata.org/wiki/Q691543","display_name":"Pace","level":2,"score":0.7418486475944519},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6603377461433411},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.623502254486084},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6080211997032166},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.5692426562309265},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.5425174832344055},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5160545706748962},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5018348693847656},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4802258610725403},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36421093344688416},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.27530378103256226},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2459251582622528},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3394112","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394112","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6310107347","display_name":null,"funder_award_id":"61872263","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":95,"referenced_works":["https://openalex.org/W1583837637","https://openalex.org/W1793186821","https://openalex.org/W1971618369","https://openalex.org/W1977556410","https://openalex.org/W1985896931","https://openalex.org/W1992987499","https://openalex.org/W1994530392","https://openalex.org/W1998393968","https://openalex.org/W2014515160","https://openalex.org/W2020538887","https://openalex.org/W2021213467","https://openalex.org/W2053873374","https://openalex.org/W2085989833","https://openalex.org/W2089468765","https://openalex.org/W2094157810","https://openalex.org/W2095873049","https://openalex.org/W2097808034","https://openalex.org/W2098203240","https://openalex.org/W2098639318","https://openalex.org/W2110527401","https://openalex.org/W2119112357","https://openalex.org/W2120563984","https://openalex.org/W2128406786","https://openalex.org/W2129761255","https://openalex.org/W2149706766","https://openalex.org/W2153342620","https://openalex.org/W2170771779","https://openalex.org/W2280162796","https://openalex.org/W2292289085","https://openalex.org/W2463553622","https://openalex.org/W2475287302","https://openalex.org/W2506015293","https://openalex.org/W2514806119","https://openalex.org/W2525984666","https://openalex.org/W2547954490","https://openalex.org/W2565516711","https://openalex.org/W2601243251","https://openalex.org/W2602856279","https://openalex.org/W2609368435","https://openalex.org/W2615067872","https://openalex.org/W2616028256","https://openalex.org/W2617809069","https://openalex.org/W2740924709","https://openalex.org/W2794497357","https://openalex.org/W2799640043","https://openalex.org/W2804337238","https://openalex.org/W2808957028","https://openalex.org/W2888307014","https://openalex.org/W2888824816","https://openalex.org/W2898868990","https://openalex.org/W2899067106","https://openalex.org/W2904507568","https://openalex.org/W2905627582","https://openalex.org/W2911897753","https://openalex.org/W2912269676","https://openalex.org/W2914982603","https://openalex.org/W2920395748","https://openalex.org/W2922015121","https://openalex.org/W2939724256","https://openalex.org/W2942544869","https://openalex.org/W2945260475","https://openalex.org/W2947133760","https://openalex.org/W2947815220","https://openalex.org/W2948254043","https://openalex.org/W2954629067","https://openalex.org/W2954678575","https://openalex.org/W2955224284","https://openalex.org/W2957905354","https://openalex.org/W2958754741","https://openalex.org/W2963058500","https://openalex.org/W2963834268","https://openalex.org/W2963857521","https://openalex.org/W2963913218","https://openalex.org/W2964164993","https://openalex.org/W2968594320","https://openalex.org/W2968940383","https://openalex.org/W2973084513","https://openalex.org/W2978547122","https://openalex.org/W3000442532","https://openalex.org/W3007157104","https://openalex.org/W3083489694","https://openalex.org/W3090608524","https://openalex.org/W3099444373","https://openalex.org/W3104570147","https://openalex.org/W3105347387","https://openalex.org/W3105599650","https://openalex.org/W4212774754","https://openalex.org/W4236137412","https://openalex.org/W4239167255","https://openalex.org/W4250967164","https://openalex.org/W4251988601","https://openalex.org/W4253639210","https://openalex.org/W4256506282","https://openalex.org/W4376850198","https://openalex.org/W6833780757"],"related_works":["https://openalex.org/W3008339103","https://openalex.org/W1667647204","https://openalex.org/W2404647514","https://openalex.org/W4247536566","https://openalex.org/W2386723501","https://openalex.org/W4241418540","https://openalex.org/W2018477250","https://openalex.org/W3119814709","https://openalex.org/W1508895727","https://openalex.org/W1578161921"],"abstract_inverted_index":{"Deep":[0],"neural":[1],"network":[2],"(DNN)":[3],"has":[4],"become":[5],"increasingly":[6],"popular":[7],"and":[8,71,85,159,277,312,323,327,330,332,335,344],"DNN":[9,24,29,49,184,310],"testing":[10,30,53,73,117,176,179,236,313,364],"is":[11,39,75,91,151,156,354],"very":[12],"critical":[13],"to":[14,41,46,145,169,200,213,225,237,256,295,356],"guarantee":[15],"the":[16,21,48,52,72,112,115,122,192,204,214,220,231,242,246,261,278,282,297,359,362,374],"correctness":[17],"of":[18,23,105,114,135,182,222,233,264,286,299,309,319,338,361],"DNN,":[19],"i.e.,":[20,37],"accuracy":[22,50,113,143,360],"in":[25,67],"this":[26,79,120,132],"work.":[27],"However,":[28],"suffers":[31],"from":[32,209,241,274],"a":[33,68,83,102,141,183,195,303],"serious":[34],"efficiency":[35],"problem,":[36,80],"it":[38,150,155],"costly":[40],"label":[42],"each":[43,57,210],"test":[44,58,106,137,172,207,239,247,265,288,339,347],"input":[45,59],"know":[47],"for":[51,93],"set,":[54],"since":[55],"labeling":[56,123,131],"involves":[60],"multiple":[61],"persons":[62],"(even":[63],"with":[64,174,366],"domain-specific":[65],"knowledge)":[66],"manual":[69],"way":[70],"set":[74,104,134,285,365],"large-scale.":[76],"To":[77],"relieve":[78],"we":[81],"propose":[82],"novel":[84],"practical":[86,149],"approach,":[87],"called":[88],"PACE":[89,147,165,190,228,300,353],"(which":[90],"short":[92],"P":[94],"ractical":[95],"AC":[96],"curacy":[97],"E":[98],"stimation),":[99],"which":[100,217],"selects":[101],"small":[103,133,284],"inputs":[107,173,240,248,340],"that":[108,154,249,352],"can":[109,125,218],"precisely":[110,357],"estimate":[111,358],"whole":[116,363],"set.":[118],"In":[119],"way,":[121],"costs":[124],"be":[126],"largely":[127],"reduced":[128],"by":[129,315],"just":[130],"selected":[136,287],"inputs.":[138,266,289],"Besides":[139],"achieving":[140],"precise":[142],"estimation,":[144],"make":[146],"more":[148],"also":[152,229],"required":[153,262],"interpretable,":[157],"deterministic,":[158],"as":[160,162],"efficient":[161],"possible.":[163],"Therefore,":[164],"first":[166],"incorporates":[167],"clustering":[168],"interpretably":[170],"divide":[171],"different":[175,180,187,317,336],"capabilities":[177],"(i.e.,":[178,203,245,272,306,321,341],"functionalities":[181],"model)":[185],"into":[186,253],"groups.":[188],"Then,":[189],"utilizes":[191],"MMD-critic":[193],"algorithm,":[194,199],"state-of-the-art":[196,375],"example-based":[197],"explanation":[198],"select":[201,238],"prototypes":[202],"most":[205],"representative":[206],"inputs)":[208],"group,":[211],"according":[212],"group":[215],"sizes,":[216],"reduce":[219],"impact":[221],"noise":[223],"due":[224],"clustering.":[226],"Meanwhile,":[227],"borrows":[230],"idea":[232],"adaptive":[234],"random":[235],"minority":[243,279],"space":[244],"are":[250],"not":[251],"clustered":[252],"any":[254],"group)":[255],"achieve":[257],"great":[258],"diversity":[259],"under":[260],"number":[263],"The":[267,349],"two":[268],"parallel":[269],"selection":[270,273],"processes":[271],"both":[275],"groups":[276],"space)":[280],"compose":[281],"final":[283],"We":[290],"conducted":[291],"an":[292],"extensive":[293],"study":[294],"evaluate":[296],"performance":[298],"based":[301],"on":[302,370],"comprehensive":[304],"benchmark":[305],"24":[307],"pairs":[308],"models":[311,320],"sets)":[314],"considering":[316],"types":[318,337],"classification":[322],"regression":[324],"models,":[325,329],"high-accuracy":[326],"low-accuracy":[328],"CNN":[331],"RNN":[333],"models)":[334],"original,":[342],"mutated,":[343],"automatically":[345],"generated":[346],"inputs).":[348],"results":[350],"demonstrate":[351],"able":[355],"only":[367],"1.181%\u223c2.302%":[368],"deviations,":[369],"average,":[371],"significantly":[372],"outperforming":[373],"approaches.":[376]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":18},{"year":2023,"cited_by_count":31},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":10},{"year":2020,"cited_by_count":3}],"updated_date":"2026-03-08T08:50:53.379069","created_date":"2025-10-10T00:00:00"}