{"id":"https://openalex.org/W2100460230","doi":"https://doi.org/10.1109/ijcnn.2008.4634255","title":"The problem with ranking ensembles based on training or validation performance","display_name":"The problem with ranking ensembles based on training or validation performance","publication_year":2008,"publication_date":"2008-06-01","ids":{"openalex":"https://openalex.org/W2100460230","doi":"https://doi.org/10.1109/ijcnn.2008.4634255","mag":"2100460230"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn.2008.4634255","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn.2008.4634255","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2008 IEEE International Joint Conference on Neural Networks (IEEE World Congress on Computational Intelligence)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:hb:diva-5946","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103036745","display_name":"Ulf Johansson","orcid":"https://orcid.org/0000-0003-0412-6199"},"institutions":[{"id":"https://openalex.org/I992397","display_name":"University of Bor\u00e5s","ror":"https://ror.org/01fdxwh83","country_code":"SE","type":"education","lineage":["https://openalex.org/I992397"]}],"countries":["SE"],"is_corresponding":true,"raw_author_name":"Ulf Johansson","raw_affiliation_strings":["School of Business and Informatics, University of Bor\u00e5s, Boras, Sweden","Sch. of Bus. & Inf., Univ. of Boras, Boras"],"affiliations":[{"raw_affiliation_string":"School of Business and Informatics, University of Bor\u00e5s, Boras, Sweden","institution_ids":["https://openalex.org/I992397"]},{"raw_affiliation_string":"Sch. of Bus. & Inf., Univ. of Boras, Boras","institution_ids":["https://openalex.org/I992397"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073489496","display_name":"Tuve L\u00f6fstr\u00f6m","orcid":"https://orcid.org/0000-0003-0274-9026"},"institutions":[{"id":"https://openalex.org/I992397","display_name":"University of Bor\u00e5s","ror":"https://ror.org/01fdxwh83","country_code":"SE","type":"education","lineage":["https://openalex.org/I992397"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Tuve Lofstrom","raw_affiliation_strings":["School of Business and Informatics, University of Bor\u00e5s, Boras, Sweden","Sch. of Bus. & Inf., Univ. of Boras, Boras"],"affiliations":[{"raw_affiliation_string":"School of Business and Informatics, University of Bor\u00e5s, Boras, Sweden","institution_ids":["https://openalex.org/I992397"]},{"raw_affiliation_string":"Sch. of Bus. & Inf., Univ. of Boras, Boras","institution_ids":["https://openalex.org/I992397"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033008105","display_name":"Henrik Bostr\u00f6m","orcid":"https://orcid.org/0000-0001-8382-0300"},"institutions":[{"id":"https://openalex.org/I205158640","display_name":"University of Sk\u00f6vde","ror":"https://ror.org/051mrsz47","country_code":"SE","type":"education","lineage":["https://openalex.org/I205158640"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Henrik Bostrom","raw_affiliation_strings":["School of Humanities and Informatics, University of Sk\u00f6vde, Sweden","Sch. of Humanities & Inf., Univ. of Skovde, Skovde"],"affiliations":[{"raw_affiliation_string":"School of Humanities and Informatics, University of Sk\u00f6vde, Sweden","institution_ids":["https://openalex.org/I205158640"]},{"raw_affiliation_string":"Sch. of Humanities & Inf., Univ. of Skovde, Skovde","institution_ids":["https://openalex.org/I205158640"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5103036745"],"corresponding_institution_ids":["https://openalex.org/I992397"],"apc_list":null,"apc_paid":null,"fwci":2.5379,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.91603458,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"18","issue":null,"first_page":"3222","last_page":"3228"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10538","display_name":"Data Mining Algorithms and Applications","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10538","display_name":"Data Mining Algorithms and Applications","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7638116478919983},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6864474415779114},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.664707601070404},{"id":"https://openalex.org/keywords/ensemble-learning","display_name":"Ensemble learning","score":0.6379872560501099},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6308099627494812},{"id":"https://openalex.org/keywords/random-subspace-method","display_name":"Random subspace method","score":0.623486340045929},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6169177889823914},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.6084790825843811},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.4911946952342987},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4884437620639801},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.46493667364120483},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.44341447949409485},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.43537914752960205},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1013580858707428}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7638116478919983},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6864474415779114},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.664707601070404},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.6379872560501099},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6308099627494812},{"id":"https://openalex.org/C106135958","wikidata":"https://www.wikidata.org/wiki/Q7291993","display_name":"Random subspace method","level":3,"score":0.623486340045929},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6169177889823914},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.6084790825843811},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.4911946952342987},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4884437620639801},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.46493667364120483},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.44341447949409485},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.43537914752960205},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1013580858707428},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/ijcnn.2008.4634255","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn.2008.4634255","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2008 IEEE International Joint Conference on Neural Networks (IEEE World Congress on Computational Intelligence)","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.140.4041","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.140.4041","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://bada.hb.se/bitstream/2320/3973/1/NN0884.pdf","raw_type":"text"},{"id":"pmh:oai:DiVA.org:hb-5946","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:hb:diva-5946","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:DiVA.org:hj-45816","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:hj:diva-45816","pdf_url":null,"source":{"id":"https://openalex.org/S4377196548","display_name":"Jonkoping University Library (J\u00f6nk\u00f6ping University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I94616838","host_organization_name":"J\u00f6nk\u00f6ping University","host_organization_lineage":["https://openalex.org/I94616838"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:DiVA.org:su-33914","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:su:diva-33914","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference paper"}],"best_oa_location":{"id":"pmh:oai:DiVA.org:hb-5946","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:hb:diva-5946","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W28412257","https://openalex.org/W1496929357","https://openalex.org/W1570448133","https://openalex.org/W2027902935","https://openalex.org/W2064620092","https://openalex.org/W2084812512","https://openalex.org/W2093717447","https://openalex.org/W2100805904","https://openalex.org/W2105658576","https://openalex.org/W2112076978","https://openalex.org/W2115629999","https://openalex.org/W2116753260","https://openalex.org/W2128073546","https://openalex.org/W2135293965","https://openalex.org/W2135724004","https://openalex.org/W2151572298","https://openalex.org/W2167055186","https://openalex.org/W2171752105","https://openalex.org/W2912934387","https://openalex.org/W3004732066","https://openalex.org/W3021787434","https://openalex.org/W4212883601","https://openalex.org/W6676769703","https://openalex.org/W6679154154"],"related_works":["https://openalex.org/W1981866886","https://openalex.org/W1964832275","https://openalex.org/W2389865566","https://openalex.org/W2052615004","https://openalex.org/W2407804800","https://openalex.org/W2388864896","https://openalex.org/W2888937984","https://openalex.org/W2046975922","https://openalex.org/W2031173026","https://openalex.org/W202723009"],"abstract_inverted_index":{"The":[0,125,148],"main":[1,126],"purpose":[2],"of":[3,41,83,144,152,192],"this":[4],"study":[5],"was":[6],"to":[7,13,22,69,133,177],"determine":[8],"whether":[9],"it":[10,174],"is":[11,53,65,90,128,175],"possible":[12],"somehow":[14,188],"use":[15],"results":[16,171],"on":[17,26,73,81],"training":[18,85,106,112,117],"or":[19],"validation":[20,108,114,119],"data":[21,75,156],"estimate":[23],"ensemble":[24,79,105,116,165,185],"performance":[25,72,186],"novel":[27,74],"data.":[28],"With":[29],"the":[30,51,84,92,95,100,142,145,170],"specific":[31],"setup":[32],"evaluated;":[33],"i.e.":[34,104],"using":[35,58],"ensembles":[36,131],"built":[37],"from":[38],"a":[39,54,155,190],"pool":[40],"independently":[42],"trained":[43],"neural":[44],"networks":[45],"and":[46,107,113,118,121],"targeting":[47],"diversity":[48,123],"only":[49],"implicitly,":[50],"answer":[52],"resounding":[55],"no.":[56],"Experimentation,":[57],"13":[59],"UCI":[60],"datasets,":[61],"shows":[62],"that":[63,94,129,154,173],"there":[64],"in":[66,71],"general":[67],"nothing":[68],"gain":[70],"by":[76,187],"choosing":[77],"an":[78,164,180],"based":[80],"any":[82],"measures":[86,96],"evaluated":[87,97],"here.":[88],"This":[89],"despite":[91],"fact":[93],"include":[98],"all":[99,130],"most":[101],"frequently":[102],"used;":[103],"accuracy,":[109,115],"base":[110,146,194],"classifier":[111],"AUC":[120],"two":[122],"measures.":[124],"reason":[127],"tend":[132],"have":[134],"quite":[135],"similar":[136],"performance,":[137],"unless":[138],"we":[139],"deliberately":[140],"lower":[141],"accuracy":[143],"classifiers.":[147,195],"key":[149],"consequence":[150],"is,":[151],"course,":[153],"miner":[157],"can":[158],"do":[159],"no":[160],"better":[161],"than":[162],"picking":[163],"at":[166,183],"random.":[167],"In":[168],"addition,":[169],"indicate":[172],"futile":[176],"look":[178],"for":[179],"algorithm":[181],"aimed":[182],"optimizing":[184],"selecting":[189],"subset":[191],"available":[193]},"counts_by_year":[{"year":2017,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}