{"id":"https://openalex.org/W7158331219","doi":"https://doi.org/10.1145/3797086","title":"Empirical Insights of Test Selection Metrics under Multiple Testing Objectives and Distribution Shifts","display_name":"Empirical Insights of Test Selection Metrics under Multiple Testing Objectives and Distribution Shifts","publication_year":2026,"publication_date":"2026-06-30","ids":{"openalex":"https://openalex.org/W7158331219","doi":"https://doi.org/10.1145/3797086"},"language":"en","primary_location":{"id":"doi:10.1145/3797086","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797086","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3797086","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134842833","display_name":"Jingyu Zhang","orcid":"https://orcid.org/0000-0001-6043-4239","observed_orcids":["https://orcid.org/0000-0001-6043-4239"]},"institutions":[{"id":"https://openalex.org/I8679417","display_name":"Hong Kong Metropolitan University","ror":"https://ror.org/0349bsm71","country_code":"HK","type":"education","lineage":["https://openalex.org/I8679417"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jingyu Zhang","raw_affiliation_strings":["Hong Kong Metropolitan University, Department of Electronic Engineering and Computer Science, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0001-6043-4239","affiliations":[{"raw_affiliation_string":"Hong Kong Metropolitan University, Department of Electronic Engineering and Computer Science, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I8679417"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134868233","display_name":"Fan Wang","orcid":"https://orcid.org/0009-0003-5444-1813","observed_orcids":["https://orcid.org/0009-0003-5444-1813"]},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Fan Wang","raw_affiliation_strings":["City University of Hong Kong, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0009-0003-5444-1813","affiliations":[{"raw_affiliation_string":"City University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051403641","display_name":"Jacky Keung","orcid":"https://orcid.org/0000-0002-3803-9600","observed_orcids":["https://orcid.org/0000-0002-3803-9600"]},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jacky Keung","raw_affiliation_strings":["City University of Hong Kong, Department of Computer Science, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-3803-9600","affiliations":[{"raw_affiliation_string":"City University of Hong Kong, Department of Computer Science, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020804635","display_name":"Yihan Liao","orcid":"https://orcid.org/0000-0002-8002-9190","observed_orcids":["https://orcid.org/0000-0002-8002-9190"]},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yihan Liao","raw_affiliation_strings":["City University of Hong Kong, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-8002-9190","affiliations":[{"raw_affiliation_string":"City University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134797190","display_name":"Yan Xiao","orcid":"https://orcid.org/0000-0002-2563-083X","observed_orcids":["https://orcid.org/0000-0002-2563-083X"]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan Xiao","raw_affiliation_strings":["Shenzhen Campus of Sun Yat-sen University, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-2563-083X","affiliations":[{"raw_affiliation_string":"Shenzhen Campus of Sun Yat-sen University, Shenzhen, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101468661","display_name":"Lei Ma","orcid":"https://orcid.org/0000-0002-8621-2420","observed_orcids":["https://orcid.org/0000-0002-8621-2420"]},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]},{"id":"https://openalex.org/I74801974","display_name":"The University of Tokyo","ror":"https://ror.org/057zh3y96","country_code":"JP","type":"education","lineage":["https://openalex.org/I74801974"]}],"countries":["CA","JP"],"is_corresponding":false,"raw_author_name":"Lei Ma","raw_affiliation_strings":["University of Alberta, Edmonton, Canada","University of Tokyo, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-8621-2420","affiliations":[{"raw_affiliation_string":"University of Alberta, Edmonton, Canada","institution_ids":["https://openalex.org/I154425047"]},{"raw_affiliation_string":"University of Tokyo, Tokyo, Japan","institution_ids":["https://openalex.org/I74801974"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.34760319,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"3","issue":"FSE","first_page":"1288","last_page":"1311"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8877999782562256,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8877999782562256,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.06539999693632126,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.01080000028014183,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5235999822616577},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4537000060081482},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.43630000948905945},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.4066999852657318},{"id":"https://openalex.org/keywords/retraining","display_name":"Retraining","score":0.39719998836517334},{"id":"https://openalex.org/keywords/feature-selection","display_name":"Feature selection","score":0.351500004529953},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.34950000047683716},{"id":"https://openalex.org/keywords/test-data","display_name":"Test data","score":0.33230000734329224}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6743000149726868},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6069999933242798},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5235999822616577},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.47690001130104065},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4584999978542328},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4537000060081482},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.43630000948905945},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4066999852657318},{"id":"https://openalex.org/C2778712577","wikidata":"https://www.wikidata.org/wiki/Q3505966","display_name":"Retraining","level":2,"score":0.39719998836517334},{"id":"https://openalex.org/C148483581","wikidata":"https://www.wikidata.org/wiki/Q446488","display_name":"Feature selection","level":2,"score":0.351500004529953},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.34950000047683716},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.33230000734329224},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3188999891281128},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.30640000104904175},{"id":"https://openalex.org/C183905921","wikidata":"https://www.wikidata.org/wiki/Q1038757","display_name":"Multiple comparisons problem","level":2,"score":0.29600000381469727},{"id":"https://openalex.org/C86469151","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Non-regression testing","level":5,"score":0.2957000136375427},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C150921843","wikidata":"https://www.wikidata.org/wiki/Q1170431","display_name":"Resampling","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.26589998602867126},{"id":"https://openalex.org/C98385598","wikidata":"https://www.wikidata.org/wiki/Q1339385","display_name":"Empirical distribution function","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.26570001244544983},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2653000056743622},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.2542000114917755}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3797086","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797086","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2604.23342","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2604.23342","pdf_url":"https://arxiv.org/pdf/2604.23342","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:pure.atira.dk:publications/6a7a803d-82b3-4d85-aed3-306df8a1bdb6","is_oa":true,"landing_page_url":"https://hdl.handle.net/2031/6a7a803d-82b3-4d85-aed3-306df8a1bdb6","pdf_url":"https://scholars.cityu.edu.hk/files/517951061/514839811.pdf","source":{"id":"https://openalex.org/S7407055387","display_name":"CityU Scholars","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Zhang, J, Fan, W, Keung, J, Liao, Y, Xiao, Y & Ma, L 2026, Empirical Insights of Test Selection Metrics under Multiple Testing Objectives and Distribution Shifts. in Proceedings of the ACM on Software Engineering. FSE edn, vol. 3, FSE058, ACM, United States, pp. 1288-1311, 34th ACM International Conference on the Foundations of Software Engineering (FSE 2026), Montreal, Quebec, Canada, 5/07/26. https://doi.org/10.1145/3797086","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3797086","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797086","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.5801785588264465,"id":"https://metadata.un.org/sdg/16"}],"x_sdgs":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2057757127","display_name":null,"funder_award_id":"62502550","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G223142494","display_name":null,"funder_award_id":"9229029","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4282291828","display_name":null,"funder_award_id":"9229029","funder_id":"https://openalex.org/F4320309893","funder_display_name":"City University of Hong Kong"},{"id":"https://openalex.org/G6392761602","display_name":null,"funder_award_id":"JP23H03372","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G7508972960","display_name":null,"funder_award_id":"JPMJCS24K8","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320309893","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23"},{"id":"https://openalex.org/F4320309949","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"},{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Deep":[0],"learning":[1],"(DL)-based":[2],"systems":[3,35],"can":[4],"exhibit":[5],"unexpected":[6],"behavior":[7],"when":[8],"exposed":[9],"to":[10,52],"out-of-distribution":[11],"(OOD)":[12],"scenarios,":[13,91,132,216],"posing":[14],"serious":[15],"risks":[16],"in":[17],"safety-critical":[18],"domains":[19],"such":[20,34],"as":[21],"malware":[22],"detection":[23],"and":[24,94,133,156,182,194,202,205,221],"autonomous":[25],"driving.":[26],"This":[27,140],"underscores":[28],"the":[29],"importance":[30],"of":[31,47,59,89,168,187],"thoroughly":[32],"testing":[33,67,128,176],"before":[36],"deployment.":[37],"To":[38,158],"this":[39,160],"end,":[40],"researchers":[41],"have":[42],"proposed":[43],"a":[44,117,218],"wide":[45],"range":[46],"test":[48,146],"selection":[49,147],"metrics":[50,60,74,124,148],"designed":[51],"effectively":[53],"select":[54],"inputs.":[55],"However,":[56],"prior":[57],"evaluations":[58],"reveal":[61],"three":[62,175,197],"key":[63],"limitations:":[64],"(1)":[65],"narrow":[66],"objectives,":[68,129],"for":[69,76,82,152],"example,":[70],"many":[71],"studies":[72],"assess":[73],"only":[75],"fault":[77],"detection,":[78,179],"leaving":[79],"their":[80,153],"effectiveness":[81],"performance":[83,180],"estimation":[84],"unclear;":[85],"(2)":[86],"limited":[87],"coverage":[88],"OOD":[90,131,188],"with":[92],"natural":[93],"label":[95,195],"shifts":[96],"are":[97,149],"rarely":[98],"considered;":[99],"(3)":[100],"Biased":[101],"dataset":[102],"selection,":[103],"where":[104],"most":[105,150],"work":[106],"focuses":[107],"on":[108],"image":[109],"data":[110,135,198],"while":[111],"other":[112],"modalities":[113,136,199],"remain":[114],"underexplored.":[115],"Consequently,":[116],"unified":[118],"benchmark":[119],"that":[120],"examines":[121],"how":[122],"these":[123],"perform":[125],"under":[126,174],"multiple":[127],"diverse":[130],"different":[134],"is":[137],"still":[138],"lacking.":[139],"leaves":[141],"practitioners":[142],"uncertain":[143],"about":[144],"which":[145],"suitable":[151],"specific":[154],"objectives":[155,177],"contexts.":[157],"address":[159],"gap,":[161],"we":[162],"conduct":[163],"an":[164],"extensive":[165],"empirical":[166],"study":[167,212],"15":[169],"existing":[170],"metrics,":[171],"evaluating":[172],"them":[173],"(fault":[178],"estimation,":[181],"retraining":[183],"guidance),":[184],"five":[185],"types":[186],"scenarios":[189],"(corrupted,":[190],"adversarial,":[191],"temporal,":[192],"natural,":[193],"shifts),":[196],"(image,":[200],"text,":[201],"Android":[203],"packages),":[204],"13":[206],"DL":[207],"models.":[208],"In":[209],"total,":[210],"our":[211],"encompasses":[213],"1,640":[214],"experimental":[215],"offering":[217],"comprehensive":[219],"evaluation":[220],"statistical":[222],"analysis.":[223]},"counts_by_year":[],"updated_date":"2026-09-26T08:19:03.415552","created_date":"2026-04-30T00:00:00"}