{"id":"https://openalex.org/W2141007997","doi":"https://doi.org/10.1186/1471-2105-7-91","title":"Bias in error estimation when using cross-validation for model selection","display_name":"Bias in error estimation when using cross-validation for model selection","alternate_titles":[],"publication_year":2006,"publication_date":"2006-02-23","ids":{"openalex":"https://openalex.org/W2141007997","doi":"https://doi.org/10.1186/1471-2105-7-91","mag":"2141007997","pmid":"https://pubmed.ncbi.nlm.nih.gov/16504092","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/1397873"},"language":"en","primary_location":{"id":"doi:10.1186/1471-2105-7-91","doi":"https://doi.org/10.1186/1471-2105-7-91","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-7-91","pdf_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-7-91","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-c","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-2","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W2141007997"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-7-91","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004831674","display_name":"Sudhir Varma","orcid":"https://orcid.org/0000-0002-4096-4782","observed_orcids":["https://orcid.org/0000-0002-4096-4782"]},"institutions":[{"id":"https://openalex.org/I4210140884","display_name":"National Cancer Institute","ror":"https://ror.org/040gcmg81","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1299303238","https://openalex.org/I4210140884"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sudhir Varma","raw_affiliation_strings":["Biometric Research Branch, National Cancer Institute, Bethesda, MD, USA. varmas@mail.nih.gov","Biometric Research Branch, National Cancer Institute, Bethesda, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Biometric Research Branch, National Cancer Institute, Bethesda, MD, USA. varmas@mail.nih.gov","institution_ids":["https://openalex.org/I4210140884"]},{"raw_affiliation_string":"Biometric Research Branch, National Cancer Institute, Bethesda, USA","institution_ids":["https://openalex.org/I4210140884"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101715307","display_name":"Richard M Simon","orcid":"https://orcid.org/0000-0003-3558-4598","observed_orcids":["https://orcid.org/0000-0003-3558-4598"]},"institutions":[{"id":"https://openalex.org/I4210140884","display_name":"National Cancer Institute","ror":"https://ror.org/040gcmg81","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1299303238","https://openalex.org/I4210140884"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Richard Simon","raw_affiliation_strings":["Biometric Research Branch, National Cancer Institute, Bethesda, MD, USA","Biometric Research Branch, National Cancer Institute, Bethesda, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Biometric Research Branch, National Cancer Institute, Bethesda, MD, USA","institution_ids":["https://openalex.org/I4210140884"]},{"raw_affiliation_string":"Biometric Research Branch, National Cancer Institute, Bethesda, USA","institution_ids":["https://openalex.org/I4210140884"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5004831674"],"corresponding_institution_ids":["https://openalex.org/I4210140884"],"apc_list":{"value":2990,"currency":"USD","value_usd":2990},"apc_paid":{"value":2990,"currency":"USD","value_usd":2990},"fwci":5.7486,"has_fulltext":true,"cited_by_count":3256,"citation_normalized_percentile":{"value":0.97594455,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"7","issue":"1","first_page":"91","last_page":"91"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.6182000041007996,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.6182000041007996,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.25999999046325684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.044199999421834946,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prediction-error-estimation","display_name":"prediction error estimation","score":0.828000009059906},{"id":"https://openalex.org/keywords/model-selection","display_name":"model selection","score":0.7540000081062317},{"id":"https://openalex.org/keywords/nested-cross-validation","display_name":"nested cross-validation","score":0.824999988079071},{"id":"https://openalex.org/keywords/classifier-parameter-tuning","display_name":"classifier parameter tuning","score":0.8159999847412109},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"support vector machine","score":0.7590000033378601},{"id":"https://openalex.org/keywords/leave-one-out-cross-validation","display_name":"leave-one-out cross-validation","score":0.9449999928474426},{"id":"https://openalex.org/keywords/10-fold-cross-validation","display_name":"10-fold cross-validation","score":0.7820000052452087}],"concepts":[{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.7273218631744385},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.6169459819793701},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.6150842308998108},{"id":"https://openalex.org/C146599234","wikidata":"https://www.wikidata.org/wiki/Q511093","display_name":"Centroid","level":2,"score":0.611228346824646},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5368976593017578},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47912144660949707},{"id":"https://openalex.org/C27181475","wikidata":"https://www.wikidata.org/wiki/Q541014","display_name":"Cross-validation","level":2,"score":0.4724954664707184},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4616066515445709},{"id":"https://openalex.org/C143809311","wikidata":"https://www.wikidata.org/wiki/Q4874458","display_name":"Bayes error rate","level":5,"score":0.44150251150131226},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.4294233024120331},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4271978735923767},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34302380681037903},{"id":"https://openalex.org/C185207860","wikidata":"https://www.wikidata.org/wiki/Q17004744","display_name":"Bayes classifier","level":4,"score":0.1232014000415802},{"id":"https://openalex.org/C52001869","wikidata":"https://www.wikidata.org/wiki/Q812530","display_name":"Naive Bayes classifier","level":3,"score":0.09628209471702576}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003198","descriptor_name":"Computer Simulation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003627","descriptor_name":"Data Interpretation, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D008957","descriptor_name":"Models, Genetic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D010363","descriptor_name":"Pattern Recognition, Automated","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D012680","descriptor_name":"Sensitivity and Specificity","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015233","descriptor_name":"Models, Statistical","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015982","descriptor_name":"Bias","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D020411","descriptor_name":"Oligonucleotide Array Sequence Analysis","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D020869","descriptor_name":"Gene Expression Profiling","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true}],"locations_count":4,"locations":[{"id":"doi:10.1186/1471-2105-7-91","doi":"https://doi.org/10.1186/1471-2105-7-91","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-7-91","pdf_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-7-91","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-c","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-2","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},{"id":"pmid:16504092","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/16504092","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC bioinformatics","raw_type":"Evaluation Study"},{"id":"pmh:oai:pubmedcentral.nih.gov:1397873","doi":null,"is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC1397873/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"Text"},{"id":"pmh:oai:doaj.org/article:58c78fff8a39483d9f6f86fc889b4380","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/58c78fff8a39483d9f6f86fc889b4380","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BMC Bioinformatics, Vol 7, Iss 1, p 91 (2006)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1186/1471-2105-7-91","doi":"https://doi.org/10.1186/1471-2105-7-91","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-7-91","pdf_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-7-91","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-c","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-2","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2141007997.pdf","grobid_xml":"https://content.openalex.org/works/W2141007997.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W1978392040","https://openalex.org/W2043235003","https://openalex.org/W2105981176","https://openalex.org/W2107956883","https://openalex.org/W2126214330","https://openalex.org/W2127314075","https://openalex.org/W2135699773","https://openalex.org/W2138550913","https://openalex.org/W2148603752","https://openalex.org/W2153635508","https://openalex.org/W2167191085","https://openalex.org/W2799061466","https://openalex.org/W4243217231"],"related_works":["https://openalex.org/W2080951248","https://openalex.org/W3092198787","https://openalex.org/W7095649830","https://openalex.org/W2067577784","https://openalex.org/W1834085549","https://openalex.org/W2155335721","https://openalex.org/W2073792037","https://openalex.org/W7005686126","https://openalex.org/W2127314075","https://openalex.org/W2314127360"],"abstract_inverted_index":{"BACKGROUND:":[0],"Cross-validation":[1],"(CV)":[2],"is":[3,164,177,220,303],"an":[4,51,160,174,181,297,335,396],"effective":[5],"method":[6],"for":[7,21,70,119,127,190,228,236,314,321,338,361],"estimating":[8,362],"the":[9,31,38,42,47,54,67,91,94,97,111,128,137,149,168,171,184,191,194,206,210,225,229,232,237,241,259,274,278,292,300,310,354,379,401],"prediction":[10],"error":[11,33,44,56,113,188,208,234,261,301,336,364],"of":[12,40,46,53,73,93,170,183,205,250,269,273,299,353,359,365,378,400],"a":[13,155,201,339,349,366,370],"classifier.":[14],"Some":[15],"recent":[16],"articles":[17],"have":[18,36],"proposed":[19],"methods":[20],"optimizing":[22],"classifiers":[23,276,320],"by":[24],"choosing":[25],"classifier":[26,49,106,192,211,340,367,382],"parameter":[27,107,383],"values":[28,108],"that":[29,109,209,302,307,330,341,375],"minimize":[30],"CV":[32,43,64,112,116,157,162,176,187,233,289,332,347,360,389,393],"estimate.":[34,114],"We":[35,62,328],"evaluated":[37],"validity":[39],"using":[41,331,346,369],"estimate":[45,52,136,182,189,204,235,298,337,352,399],"optimized":[48,275],"as":[50],"true":[55,138,207,355,363,402],"expected":[57],"on":[58,214,248,267,277,309],"independent":[59,215,279,311],"data.":[60,216],"RESULTS:":[61],"used":[63,118,126,165,178],"to":[65,135,166,179,199,306,333],"optimize":[66],"classification":[68],"parameters":[69,172,196,243,258],"two":[71,98,226],"kinds":[72],"classifiers;":[74],"Shrunken":[75,120,238,316],"Centroids":[76,121,317],"and":[77,142,295,318,323],"Support":[78],"Vector":[79],"Machines":[80],"(SVM).":[81],"Random":[82],"training":[83,252],"datasets":[84],"were":[85],"created,":[86],"with":[87,193,240,256],"no":[88,221,283],"difference":[89,223],"in":[90,387],"distribution":[92],"features":[95],"between":[96,148,224],"classes.":[99],"Using":[100],"these":[101],"\"null\"":[102,141,230,270,322],"datasets,":[103,231],"we":[104,152],"selected":[105],"minimized":[110],"10-fold":[115],"was":[117,125,133,197,244,263,282],"while":[122,173],"Leave-One-Out-CV":[123],"(LOOCV)":[124],"SVM.":[129],"Independent":[130],"test":[131,280],"data":[132,325],"created":[134],"error.":[139,185,356,403],"With":[140],"\"non":[143],"null\"":[144],"(with":[145],"differential":[146],"expression":[147],"classes)":[150],"data,":[151],"also":[153],"tested":[154],"nested":[156,288,392],"procedure,":[158],"where":[159],"inner":[161],"loop":[163],"perform":[167],"tuning":[169],"outer":[175],"compute":[180,334],"The":[186,287],"optimal":[195,242,257],"found":[198],"be":[200,385],"substantially":[202],"biased":[203,351],"would":[212],"incur":[213],"Even":[217],"though":[218],"there":[219],"real":[222],"classes":[227],"Centroid":[239],"less":[245,264],"than":[246,265,285],"30%":[247,266],"18.5%":[249],"simulated":[251],"data-sets.":[253,271],"For":[254],"SVM":[255,319],"estimated":[260],"rate":[262],"38%":[268],"Performance":[272],"set":[281,313],"better":[284],"chance.":[286],"procedure":[290,394],"reduces":[291],"bias":[293],"considerably":[294],"gives":[296,348],"very":[304],"close":[305],"obtained":[308],"testing":[312],"both":[315],"\"non-null\"":[324],"distributions.":[326],"CONCLUSION:":[327],"show":[329],"has":[342],"itself":[343],"been":[344],"tuned":[345],"significantly":[350],"Proper":[357],"use":[358],"developed":[368],"well":[371],"defined":[372],"algorithm":[373],"requires":[374],"all":[376],"steps":[377],"algorithm,":[380],"including":[381],"tuning,":[384],"repeated":[386],"each":[388],"loop.":[390],"A":[391],"provides":[395],"almost":[397],"unbiased":[398]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2141007997","counts_by_year":[{"year":2026,"cited_by_count":1310},{"year":2025,"cited_by_count":304},{"year":2024,"cited_by_count":171},{"year":2023,"cited_by_count":183},{"year":2022,"cited_by_count":199},{"year":2021,"cited_by_count":196},{"year":2020,"cited_by_count":204},{"year":2019,"cited_by_count":125},{"year":2018,"cited_by_count":110},{"year":2017,"cited_by_count":100},{"year":2016,"cited_by_count":59},{"year":2015,"cited_by_count":51},{"year":2014,"cited_by_count":46},{"year":2013,"cited_by_count":44},{"year":2012,"cited_by_count":38}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2016-06-24T00:00:00"}
