{"id":"https://openalex.org/W3081847095","doi":"https://doi.org/10.3390/make2030018","title":"Beyond Cross-Validation\u2014Accuracy Estimation for Incremental and Active Learning Models","display_name":"Beyond Cross-Validation\u2014Accuracy Estimation for Incremental and Active Learning Models","publication_year":2020,"publication_date":"2020-09-01","ids":{"openalex":"https://openalex.org/W3081847095","doi":"https://doi.org/10.3390/make2030018","mag":"3081847095"},"language":"en","primary_location":{"id":"doi:10.3390/make2030018","is_oa":true,"landing_page_url":"https://doi.org/10.3390/make2030018","pdf_url":"https://www.mdpi.com/2504-4990/2/3/18/pdf?version=1600831795","source":{"id":"https://openalex.org/S4210213891","display_name":"Machine Learning and Knowledge Extraction","issn_l":"2504-4990","issn":["2504-4990"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning and Knowledge Extraction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2504-4990/2/3/18/pdf?version=1600831795","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016207187","display_name":"Christian Limberg","orcid":"https://orcid.org/0000-0002-4903-3933"},"institutions":[{"id":"https://openalex.org/I20121455","display_name":"Bielefeld University","ror":"https://ror.org/02hpadn98","country_code":"DE","type":"education","lineage":["https://openalex.org/I20121455"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Christian Limberg","raw_affiliation_strings":["HONDA Research Institute Europe GmbH, 63073 Offenbach, Germany","Research Institute for Cognition and Robotics, Bielefeld University, 33615 Bielefeld, Germany"],"affiliations":[{"raw_affiliation_string":"HONDA Research Institute Europe GmbH, 63073 Offenbach, Germany","institution_ids":[]},{"raw_affiliation_string":"Research Institute for Cognition and Robotics, Bielefeld University, 33615 Bielefeld, Germany","institution_ids":["https://openalex.org/I20121455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107396066","display_name":"Heiko Wersing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Heiko Wersing","raw_affiliation_strings":["HONDA Research Institute Europe GmbH, 63073 Offenbach, Germany"],"affiliations":[{"raw_affiliation_string":"HONDA Research Institute Europe GmbH, 63073 Offenbach, Germany","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085189928","display_name":"Helge Ritter","orcid":null},"institutions":[{"id":"https://openalex.org/I20121455","display_name":"Bielefeld University","ror":"https://ror.org/02hpadn98","country_code":"DE","type":"education","lineage":["https://openalex.org/I20121455"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Helge Ritter","raw_affiliation_strings":["Research Institute for Cognition and Robotics, Bielefeld University, 33615 Bielefeld, Germany"],"affiliations":[{"raw_affiliation_string":"Research Institute for Cognition and Robotics, Bielefeld University, 33615 Bielefeld, Germany","institution_ids":["https://openalex.org/I20121455"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5016207187"],"corresponding_institution_ids":["https://openalex.org/I20121455"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":0.9279,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.80501707,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"2","issue":"3","first_page":"327","last_page":"346"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7406262159347534},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7162042260169983},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6904687881469727},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.6228371262550354},{"id":"https://openalex.org/keywords/supervised-learning","display_name":"Supervised learning","score":0.5273740291595459},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4724307954311371},{"id":"https://openalex.org/keywords/cross-validation","display_name":"Cross-validation","score":0.4534800052642822},{"id":"https://openalex.org/keywords/regression","display_name":"Regression","score":0.4106155037879944},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.38845402002334595},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3283340334892273},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.18331307172775269},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08880412578582764},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07609882950782776}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7406262159347534},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7162042260169983},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6904687881469727},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.6228371262550354},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.5273740291595459},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4724307954311371},{"id":"https://openalex.org/C27181475","wikidata":"https://www.wikidata.org/wiki/Q541014","display_name":"Cross-validation","level":2,"score":0.4534800052642822},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.4106155037879944},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.38845402002334595},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3283340334892273},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.18331307172775269},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08880412578582764},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07609882950782776},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3390/make2030018","is_oa":true,"landing_page_url":"https://doi.org/10.3390/make2030018","pdf_url":"https://www.mdpi.com/2504-4990/2/3/18/pdf?version=1600831795","source":{"id":"https://openalex.org/S4210213891","display_name":"Machine Learning and Knowledge Extraction","issn_l":"2504-4990","issn":["2504-4990"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning and Knowledge Extraction","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:24c6c562fb1b49df8271b9361fdbd5a3","is_oa":true,"landing_page_url":"https://doaj.org/article/24c6c562fb1b49df8271b9361fdbd5a3","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Machine Learning and Knowledge Extraction, Vol 2, Iss 3, Pp 327-346 (2020)","raw_type":"article"},{"id":"pmh:oai:mdpi.com:/2504-4990/2/3/18/","is_oa":true,"landing_page_url":"https://dx.doi.org/10.3390/make2030018","pdf_url":null,"source":{"id":"https://openalex.org/S4306400947","display_name":"MDPI (MDPI AG)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210097602","host_organization_name":"Multidisciplinary Digital Publishing Institute (Switzerland)","host_organization_lineage":["https://openalex.org/I4210097602"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Machine Learning and Knowledge Extraction; Volume 2; Issue 3; Pages: 327-346","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.3390/make2030018","is_oa":true,"landing_page_url":"https://doi.org/10.3390/make2030018","pdf_url":"https://www.mdpi.com/2504-4990/2/3/18/pdf?version=1600831795","source":{"id":"https://openalex.org/S4210213891","display_name":"Machine Learning and Knowledge Extraction","issn_l":"2504-4990","issn":["2504-4990"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning and Knowledge Extraction","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7300000190734863}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3081847095.pdf","grobid_xml":"https://content.openalex.org/works/W3081847095.grobid-xml"},"referenced_works_count":50,"referenced_works":["https://openalex.org/W1006028607","https://openalex.org/W1519697908","https://openalex.org/W1618905105","https://openalex.org/W1656892363","https://openalex.org/W1686810756","https://openalex.org/W1797761079","https://openalex.org/W1834627138","https://openalex.org/W1932070659","https://openalex.org/W1982039810","https://openalex.org/W1985602045","https://openalex.org/W1990863955","https://openalex.org/W2010158189","https://openalex.org/W2076333696","https://openalex.org/W2080021732","https://openalex.org/W2088220893","https://openalex.org/W2094412673","https://openalex.org/W2099906136","https://openalex.org/W2123749980","https://openalex.org/W2139990538","https://openalex.org/W2155904486","https://openalex.org/W2160512933","https://openalex.org/W2171585602","https://openalex.org/W2186094539","https://openalex.org/W2282821441","https://openalex.org/W2502759836","https://openalex.org/W2620661538","https://openalex.org/W2739068567","https://openalex.org/W2756359217","https://openalex.org/W2758219826","https://openalex.org/W2788388592","https://openalex.org/W2891503716","https://openalex.org/W2903092938","https://openalex.org/W2903158431","https://openalex.org/W2911964244","https://openalex.org/W2942157335","https://openalex.org/W2945526235","https://openalex.org/W2963991843","https://openalex.org/W2972897192","https://openalex.org/W2981731882","https://openalex.org/W2997967197","https://openalex.org/W2999615587","https://openalex.org/W4226065182","https://openalex.org/W4239510810","https://openalex.org/W6626187645","https://openalex.org/W6636501900","https://openalex.org/W6678524678","https://openalex.org/W6685112792","https://openalex.org/W6741753795","https://openalex.org/W6751800415","https://openalex.org/W6993175652"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2048917867","https://openalex.org/W3130163047","https://openalex.org/W2902329723","https://openalex.org/W1800458610","https://openalex.org/W2384527366","https://openalex.org/W2917343039","https://openalex.org/W3011325431"],"abstract_inverted_index":{"For":[0],"incremental":[1,112],"machine-learning":[2],"applications":[3],"it":[4,72],"is":[5,73],"often":[6],"important":[7],"to":[8,55,75,147,162],"robustly":[9],"estimate":[10],"the":[11,20,30,50,57,84,138,142,155],"system":[12],"accuracy":[13,39,58,86,124],"during":[14],"training,":[15],"especially":[16],"if":[17],"humans":[18],"perform":[19],"supervised":[21,32,132],"teaching.":[22],"Cross-validation":[23],"and":[24,102,105,113,129],"interleaved":[25],"test/train":[26],"error":[27],"are":[28],"here":[29],"standard":[31,127],"approaches.":[33],"We":[34,48,94,140],"propose":[35],"a":[36,91,148],"novel":[37,88,121],"semi-supervised":[38,92],"estimation":[40,125],"approach":[41,54,146],"that":[42,62,119],"clearly":[43],"outperforms":[44],"these":[45],"two":[46],"methods.":[47],"introduce":[49],"Configram":[51],"Estimation":[52],"(CGEM)":[53],"predict":[56],"of":[59,82,137,144],"any":[60],"classifier":[61],"delivers":[63],"confidences.":[64],"By":[65],"calculating":[66],"classification":[67],"confidences":[68],"for":[69,110],"unseen":[70],"samples,":[71],"possible":[74],"train":[76],"an":[77],"offline":[78],"regression":[79],"model,":[80],"capable":[81],"predicting":[83],"classifier\u2019s":[85],"on":[87,103],"data":[89,108,134],"in":[90],"fashion.":[93],"evaluate":[95],"our":[96,120,145,160],"method":[97,122,161],"with":[98],"several":[99],"diverse":[100],"classifiers":[101],"analytical":[104],"real-world":[106],"benchmark":[107],"sets":[109],"both":[111],"active":[114],"learning.":[115],"The":[116],"results":[117],"show":[118],"improves":[123],"over":[126],"methods":[128],"requires":[130],"less":[131],"training":[133],"after":[135],"deployment":[136],"model.":[139],"demonstrate":[141],"application":[143],"challenging":[149],"robot":[150],"object":[151],"recognition":[152],"task,":[153],"where":[154],"human":[156],"teacher":[157],"can":[158],"use":[159],"judge":[163],"sufficient":[164],"training.":[165]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}