{"id":"https://openalex.org/W2474835145","doi":"https://doi.org/10.1109/tse.2016.2584050","title":"An Empirical Comparison of Model Validation Techniques for Defect Prediction Models","display_name":"An Empirical Comparison of Model Validation Techniques for Defect Prediction Models","publication_year":2016,"publication_date":"2016-06-23","ids":{"openalex":"https://openalex.org/W2474835145","doi":"https://doi.org/10.1109/tse.2016.2584050","mag":"2474835145"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2016.2584050","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2016.2584050","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081449581","display_name":"Chakkrit Tantithamthavorn","orcid":"https://orcid.org/0000-0002-5516-9984"},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Chakkrit Tantithamthavorn","raw_affiliation_strings":["Graduate School of Information Science, Nara Institute of Science and Technology, Ikoma, Japan"],"affiliations":[{"raw_affiliation_string":"Graduate School of Information Science, Nara Institute of Science and Technology, Ikoma, Japan","institution_ids":["https://openalex.org/I75917431"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043053174","display_name":"Shane McIntosh","orcid":"https://orcid.org/0000-0002-0193-3975"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shane McIntosh","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Montreal, QC, Canada"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Montreal, QC, Canada","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091586373","display_name":"Ahmed E. Hassan","orcid":"https://orcid.org/0000-0001-7749-5513"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ahmed E. Hassan","raw_affiliation_strings":["School of Computing, Queen\u2019s University, Kingston, ON, Canada","School of Computing, Queen's University, Kingston, ON, Canada"],"affiliations":[{"raw_affiliation_string":"School of Computing, Queen\u2019s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"School of Computing, Queen's University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011588138","display_name":"Kenichi Matsumoto","orcid":"https://orcid.org/0000-0002-7418-9323"},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kenichi Matsumoto","raw_affiliation_strings":["Graduate School of Information Science, Nara Institute of Science and Technology, Ikoma, Japan"],"affiliations":[{"raw_affiliation_string":"Graduate School of Information Science, Nara Institute of Science and Technology, Ikoma, Japan","institution_ids":["https://openalex.org/I75917431"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5081449581"],"corresponding_institution_ids":["https://openalex.org/I75917431"],"apc_list":null,"apc_paid":null,"fwci":99.7718,"has_fulltext":false,"cited_by_count":557,"citation_normalized_percentile":{"value":0.99949005,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"43","issue":"1","first_page":"1","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7560739517211914},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.6445061564445496},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5687437653541565},{"id":"https://openalex.org/keywords/cross-validation","display_name":"Cross-validation","score":0.5077310800552368},{"id":"https://openalex.org/keywords/model-validation","display_name":"Model validation","score":0.48728859424591064},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.4706539213657379},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.46000486612319946},{"id":"https://openalex.org/keywords/predictive-modelling","display_name":"Predictive modelling","score":0.4469338357448578},{"id":"https://openalex.org/keywords/software-bug","display_name":"Software bug","score":0.4344905912876129},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3654106855392456},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3313683867454529}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7560739517211914},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6445061564445496},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5687437653541565},{"id":"https://openalex.org/C27181475","wikidata":"https://www.wikidata.org/wiki/Q541014","display_name":"Cross-validation","level":2,"score":0.5077310800552368},{"id":"https://openalex.org/C3019813237","wikidata":"https://www.wikidata.org/wiki/Q65089264","display_name":"Model validation","level":2,"score":0.48728859424591064},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4706539213657379},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.46000486612319946},{"id":"https://openalex.org/C45804977","wikidata":"https://www.wikidata.org/wiki/Q7239673","display_name":"Predictive modelling","level":2,"score":0.4469338357448578},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.4344905912876129},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3654106855392456},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3313683867454529},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2016.2584050","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2016.2584050","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.4300000071525574,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G2863578014","display_name":null,"funder_award_id":"16J03360","funder_id":"https://openalex.org/F4320320212","funder_display_name":"Japan Society for the Promotion of Science London"}],"funders":[{"id":"https://openalex.org/F4320314000","display_name":"Compute Canada","ror":"https://ror.org/03ty8yr27"},{"id":"https://openalex.org/F4320320212","display_name":"Japan Society for the Promotion of Science London","ror":"https://ror.org/02m7axw05"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"},{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":143,"referenced_works":["https://openalex.org/W25935857","https://openalex.org/W28412257","https://openalex.org/W129305155","https://openalex.org/W157748702","https://openalex.org/W168970045","https://openalex.org/W184219092","https://openalex.org/W353957520","https://openalex.org/W1480376833","https://openalex.org/W1516193414","https://openalex.org/W1554055400","https://openalex.org/W1680392829","https://openalex.org/W1831050183","https://openalex.org/W1920585257","https://openalex.org/W1964544799","https://openalex.org/W1971654961","https://openalex.org/W1973009402","https://openalex.org/W1975040830","https://openalex.org/W1976526581","https://openalex.org/W1976927254","https://openalex.org/W1981251392","https://openalex.org/W1984425068","https://openalex.org/W1993555269","https://openalex.org/W1994493193","https://openalex.org/W1995945562","https://openalex.org/W1999830936","https://openalex.org/W2005504865","https://openalex.org/W2007705030","https://openalex.org/W2019338079","https://openalex.org/W2020181407","https://openalex.org/W2021688474","https://openalex.org/W2024623357","https://openalex.org/W2025700486","https://openalex.org/W2028896946","https://openalex.org/W2033418259","https://openalex.org/W2037557484","https://openalex.org/W2037668591","https://openalex.org/W2039081830","https://openalex.org/W2040615655","https://openalex.org/W2046830558","https://openalex.org/W2057328179","https://openalex.org/W2060374620","https://openalex.org/W2063876764","https://openalex.org/W2067490448","https://openalex.org/W2068430427","https://openalex.org/W2068816944","https://openalex.org/W2070567333","https://openalex.org/W2072785870","https://openalex.org/W2073241381","https://openalex.org/W2073792037","https://openalex.org/W2074218040","https://openalex.org/W2082183626","https://openalex.org/W2084085041","https://openalex.org/W2091498228","https://openalex.org/W2094104004","https://openalex.org/W2094764356","https://openalex.org/W2097883090","https://openalex.org/W2098673848","https://openalex.org/W2100697007","https://openalex.org/W2100945416","https://openalex.org/W2103296684","https://openalex.org/W2105776892","https://openalex.org/W2105981176","https://openalex.org/W2107031757","https://openalex.org/W2107641306","https://openalex.org/W2108191500","https://openalex.org/W2109325327","https://openalex.org/W2110298216","https://openalex.org/W2111814036","https://openalex.org/W2112081648","https://openalex.org/W2112658968","https://openalex.org/W2114105368","https://openalex.org/W2115709314","https://openalex.org/W2118978333","https://openalex.org/W2118988523","https://openalex.org/W2119910794","https://openalex.org/W2122702470","https://openalex.org/W2123279272","https://openalex.org/W2124172959","https://openalex.org/W2124545752","https://openalex.org/W2126436234","https://openalex.org/W2126957654","https://openalex.org/W2127314075","https://openalex.org/W2129925362","https://openalex.org/W2135198476","https://openalex.org/W2137122305","https://openalex.org/W2138428785","https://openalex.org/W2138827327","https://openalex.org/W2139749383","https://openalex.org/W2139855137","https://openalex.org/W2140785063","https://openalex.org/W2142481192","https://openalex.org/W2145236186","https://openalex.org/W2145574830","https://openalex.org/W2145680370","https://openalex.org/W2146338950","https://openalex.org/W2146474553","https://openalex.org/W2148371346","https://openalex.org/W2149014999","https://openalex.org/W2151666086","https://openalex.org/W2154446978","https://openalex.org/W2157825442","https://openalex.org/W2159747233","https://openalex.org/W2160958420","https://openalex.org/W2163018352","https://openalex.org/W2163732854","https://openalex.org/W2163837601","https://openalex.org/W2164627280","https://openalex.org/W2166573308","https://openalex.org/W2172232422","https://openalex.org/W2218021565","https://openalex.org/W2297716127","https://openalex.org/W2330210193","https://openalex.org/W2367798545","https://openalex.org/W2408574027","https://openalex.org/W2589382121","https://openalex.org/W2787894218","https://openalex.org/W2911964244","https://openalex.org/W2987883775","https://openalex.org/W3106889297","https://openalex.org/W3141989311","https://openalex.org/W4206600618","https://openalex.org/W4236586490","https://openalex.org/W4240402692","https://openalex.org/W4241607140","https://openalex.org/W4241672963","https://openalex.org/W4243379839","https://openalex.org/W4243562335","https://openalex.org/W4249991467","https://openalex.org/W4250023757","https://openalex.org/W4250236131","https://openalex.org/W4285719527","https://openalex.org/W4302596785","https://openalex.org/W4399539398","https://openalex.org/W6607567296","https://openalex.org/W6631129480","https://openalex.org/W6633076095","https://openalex.org/W6637386731","https://openalex.org/W6641210023","https://openalex.org/W6674712254","https://openalex.org/W6676989766","https://openalex.org/W6677507105","https://openalex.org/W6713589489","https://openalex.org/W6734155205"],"related_works":["https://openalex.org/W816105089","https://openalex.org/W2100523380","https://openalex.org/W4318240167","https://openalex.org/W3048572280","https://openalex.org/W3011444647","https://openalex.org/W2514173981","https://openalex.org/W2040236930","https://openalex.org/W3207949591","https://openalex.org/W4253742790","https://openalex.org/W2095987352"],"abstract_inverted_index":{"Defect":[0],"prediction":[1,119,196],"models":[2],"help":[3],"software":[4],"quality":[5],"assurance":[6],"teams":[7],"to":[8,13,31,56,92,147],"allocate":[9],"their":[10],"limited":[11],"resources":[12],"the":[14,40,49,63,72,117,123,161,167,174,178,185],"most":[15,125],"defect-prone":[16],"modules.":[17],"Model":[18],"validation":[19,53,69,101,129,145,164,172],"techniques,":[20],"such":[21],"as":[22],"<inline-formula><tex-math":[23],"notation=\"LaTeX\">$k$</tex-math>":[24],"</inline-formula>":[25],"-fold":[26],"cross-validation,":[27],"use":[28,204],"historical":[29],"data":[30],"estimate":[32],"how":[33,47],"well":[34],"a":[35,104,134],"model":[36,52,68,100,128,163],"will":[37],"perform":[38],"in":[39,71,116,184],"future.":[41],"However,":[42],"little":[43],"is":[44,103],"known":[45],"about":[46],"accurate":[48],"estimates":[50,149,183],"of":[51,67,74,78,87,113,137,182,187],"techniques":[54,70,130],"tend":[55],"be.":[57],"In":[58],"this":[59],"paper,":[60],"we":[61,121,140,191],"investigate":[62],"bias":[64,154,179],"and":[65,155,180,202],"variance":[66,159,181],"domain":[73],"defect":[75,81,118,195],"prediction.":[76],"Analysis":[77],"101":[79],"public":[80],"datasets":[82],"suggests":[83],"that":[84,142,193],"77":[85],"percent":[86,152,157],"them":[88],"are":[89],"highly":[90],"susceptible":[91],"producing":[93],"unstable":[94],"results\u2013":[95],"-":[96],"selecting":[97],"an":[98,111],"appropriate":[99],"technique":[102],"critical":[105],"experimental":[106],"design":[107],"choice.":[108],"Based":[109],"on":[110],"analysis":[112],"256":[114],"studies":[115,197],"literature,":[120],"select":[122],"12":[124],"commonly":[126],"adopted":[127],"for":[131],"evaluation.":[132],"Through":[133],"case":[135],"study":[136],"18":[138],"systems,":[139],"find":[141],"single-repetition":[143,199],"holdout":[144,200],"tends":[146],"produce":[148],"with":[150],"46-229":[151],"more":[153,158],"53-863":[156],"than":[160],"top-ranked":[162],"techniques.":[165],"On":[166],"other":[168],"hand,":[169],"out-of-sample":[170,205],"bootstrap":[171,206],"yields":[173],"best":[175],"balance":[176],"between":[177],"context":[186],"our":[188],"study.":[189],"Therefore,":[190],"recommend":[192],"future":[194],"avoid":[198],"validation,":[201],"instead,":[203],"validation.":[207]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":51},{"year":2024,"cited_by_count":70},{"year":2023,"cited_by_count":61},{"year":2022,"cited_by_count":57},{"year":2021,"cited_by_count":99},{"year":2020,"cited_by_count":64},{"year":2019,"cited_by_count":75},{"year":2018,"cited_by_count":46},{"year":2017,"cited_by_count":18},{"year":2016,"cited_by_count":5}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}