{"id":"https://openalex.org/W3173017111","doi":"https://doi.org/10.1007/s10994-021-06003-9","title":"How to measure uncertainty in uncertainty sampling for active learning","display_name":"How to measure uncertainty in uncertainty sampling for active learning","publication_year":2021,"publication_date":"2021-06-18","ids":{"openalex":"https://openalex.org/W3173017111","doi":"https://doi.org/10.1007/s10994-021-06003-9","mag":"3173017111"},"language":"en","primary_location":{"id":"doi:10.1007/s10994-021-06003-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10994-021-06003-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-06003-9.pdf","source":{"id":"https://openalex.org/S62148650","display_name":"Machine Learning","issn_l":"0885-6125","issn":["0885-6125","1573-0565"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-06003-9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083724281","display_name":"Vu-Linh Nguyen","orcid":"https://orcid.org/0000-0003-1642-4468"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Vu-Linh Nguyen","raw_affiliation_strings":["Department of Mathematics and Computer Science, Eindhoven University of Technology, Eindhoven, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Mathematics and Computer Science, Eindhoven University of Technology, Eindhoven, The Netherlands","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111483488","display_name":"Mohammad Shaker","orcid":null},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Mohammad Hossein Shaker","raw_affiliation_strings":["Heinz Nixdorf Institute and Department of Computer Science, Paderborn University, Paderborn, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Heinz Nixdorf Institute and Department of Computer Science, Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059439673","display_name":"Eyke H\u00fcllermeier","orcid":"https://orcid.org/0000-0002-9944-4108"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Eyke H\u00fcllermeier","raw_affiliation_strings":["Heinz Nixdorf Institute and Department of Computer Science, Paderborn University, Paderborn, Germany"],"raw_orcid":"https://orcid.org/0000-0002-9944-4108","affiliations":[{"raw_affiliation_string":"Heinz Nixdorf Institute and Department of Computer Science, Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5059439673"],"corresponding_institution_ids":["https://openalex.org/I206945453"],"apc_list":{"value":2990,"currency":"USD","value_usd":2990},"apc_paid":{"value":2990,"currency":"USD","value_usd":2990},"fwci":13.7406,"has_fulltext":true,"cited_by_count":194,"citation_normalized_percentile":{"value":0.9906293,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"111","issue":"1","first_page":"89","last_page":"122"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9871000051498413,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9632999897003174,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6365163922309875},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6246220469474792},{"id":"https://openalex.org/keywords/sensitivity-analysis","display_name":"Sensitivity analysis","score":0.5993156433105469},{"id":"https://openalex.org/keywords/uncertainty-quantification","display_name":"Uncertainty quantification","score":0.5811735391616821},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5739864706993103},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5680961608886719},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.5486416220664978},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.534065306186676},{"id":"https://openalex.org/keywords/uncertainty-analysis","display_name":"Uncertainty analysis","score":0.5302077531814575},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5169326663017273},{"id":"https://openalex.org/keywords/measurement-uncertainty","display_name":"Measurement uncertainty","score":0.47606155276298523},{"id":"https://openalex.org/keywords/active-learning","display_name":"Active learning (machine learning)","score":0.4431758224964142},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3212825655937195},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.28813308477401733},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.20840051770210266}],"concepts":[{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6365163922309875},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6246220469474792},{"id":"https://openalex.org/C176147448","wikidata":"https://www.wikidata.org/wiki/Q1889114","display_name":"Sensitivity analysis","level":3,"score":0.5993156433105469},{"id":"https://openalex.org/C32230216","wikidata":"https://www.wikidata.org/wiki/Q7882499","display_name":"Uncertainty quantification","level":2,"score":0.5811735391616821},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5739864706993103},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5680961608886719},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.5486416220664978},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.534065306186676},{"id":"https://openalex.org/C177803969","wikidata":"https://www.wikidata.org/wiki/Q29205","display_name":"Uncertainty analysis","level":2,"score":0.5302077531814575},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5169326663017273},{"id":"https://openalex.org/C137209882","wikidata":"https://www.wikidata.org/wiki/Q1403517","display_name":"Measurement uncertainty","level":2,"score":0.47606155276298523},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.4431758224964142},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3212825655937195},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28813308477401733},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.20840051770210266},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1007/s10994-021-06003-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10994-021-06003-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-06003-9.pdf","source":{"id":"https://openalex.org/S62148650","display_name":"Machine Learning","issn_l":"0885-6125","issn":["0885-6125","1573-0565"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning","raw_type":"journal-article"},{"id":"pmh:oai:epub.ub.uni-muenchen.de:91888","is_oa":false,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bvb:19-epub-91888-0","pdf_url":null,"source":{"id":"https://openalex.org/S4306401845","display_name":"Open access LMU (Ludwid Maxmilian's Universitat Munchen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I8204097","host_organization_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","host_organization_lineage":["https://openalex.org/I8204097"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Machine Learning","raw_type":"doc-type:article"},{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/4c458387-9335-49b8-9bb0-8a1d0e62d9dc","is_oa":true,"landing_page_url":"https://research.tue.nl/en/publications/4c458387-9335-49b8-9bb0-8a1d0e62d9dc","pdf_url":"https://pure.tue.nl/ws/files/195860905/Nguyen2022_Article_HowToMeasureUncertaintyInUncer.pdf","source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Nguyen, V-L, Shaker, M H & H\u00fcllermeier, E 2022, 'How to measure uncertainty in uncertainty sampling for active learning', Machine Learning, vol. 111, pp. 89-122. https://doi.org/10.1007/s10994-021-06003-9","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:tue:oai:pure.tue.nl:publications/4c458387-9335-49b8-9bb0-8a1d0e62d9dc","is_oa":true,"landing_page_url":"https://research.tue.nl/nl/publications/4c458387-9335-49b8-9bb0-8a1d0e62d9dc","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Machine Learning, 111, 89 - 122. Springer","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1007/s10994-021-06003-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10994-021-06003-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-06003-9.pdf","source":{"id":"https://openalex.org/S62148650","display_name":"Machine Learning","issn_l":"0885-6125","issn":["0885-6125","1573-0565"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6899999976158142,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G7723709779","display_name":"Multilabel Regellernen","funder_award_id":"400845550","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320323015","display_name":"Bundesministerium f\u00fcr Forschung und Technologie","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320324816","display_name":"Universit\u00e4t Paderborn","ror":"https://ror.org/058kzsd48"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3173017111.pdf","grobid_xml":"https://content.openalex.org/works/W3173017111.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W206794860","https://openalex.org/W586780102","https://openalex.org/W1514707997","https://openalex.org/W1548567853","https://openalex.org/W1554944419","https://openalex.org/W1569099997","https://openalex.org/W1979412670","https://openalex.org/W1986610361","https://openalex.org/W1986760411","https://openalex.org/W2006715371","https://openalex.org/W2012097495","https://openalex.org/W2021732807","https://openalex.org/W2039643849","https://openalex.org/W2049886280","https://openalex.org/W2056763477","https://openalex.org/W2067438012","https://openalex.org/W2069266605","https://openalex.org/W2080021732","https://openalex.org/W2085989833","https://openalex.org/W2089224781","https://openalex.org/W2110018596","https://openalex.org/W2112702040","https://openalex.org/W2119408169","https://openalex.org/W2122111042","https://openalex.org/W2125283600","https://openalex.org/W2140733639","https://openalex.org/W2149298154","https://openalex.org/W2149706766","https://openalex.org/W2158216009","https://openalex.org/W2169027522","https://openalex.org/W2171506566","https://openalex.org/W2171671120","https://openalex.org/W2262342046","https://openalex.org/W2293618063","https://openalex.org/W2330520560","https://openalex.org/W2342049278","https://openalex.org/W2788755844","https://openalex.org/W2903158431","https://openalex.org/W2950517871","https://openalex.org/W2951911250","https://openalex.org/W2980423179","https://openalex.org/W2982318248","https://openalex.org/W3014596384","https://openalex.org/W3018928030","https://openalex.org/W3029645440","https://openalex.org/W3134774296","https://openalex.org/W4240824175","https://openalex.org/W4301003819","https://openalex.org/W4301014524","https://openalex.org/W6634277557","https://openalex.org/W6680760476","https://openalex.org/W6756615331","https://openalex.org/W6817491142","https://openalex.org/W6842097421"],"related_works":["https://openalex.org/W4205181462","https://openalex.org/W3212153563","https://openalex.org/W4249135812","https://openalex.org/W2116089868","https://openalex.org/W4312671192","https://openalex.org/W2376953431","https://openalex.org/W93577605","https://openalex.org/W2387053421","https://openalex.org/W2607177949","https://openalex.org/W4386584851"],"abstract_inverted_index":{"Abstract":[0],"Various":[1],"strategies":[2,161],"for":[3,34,134],"active":[4,25,143],"learning":[5,12],"have":[6,79],"been":[7,80],"proposed":[8,81],"in":[9,71,82,117,142,167],"the":[10,20,24,29,47,52,105,109,114,129,156,159],"machine":[11,72],"literature.":[13],"In":[14,85],"uncertainty":[15,70,77,116,135,150],"sampling,":[16,136],"which":[17,35],"is":[18,39,125],"among":[19],"most":[21],"popular":[22],"approaches,":[23],"learner":[26],"sequentially":[27],"queries":[28],"label":[30],"of":[31,54,61,88,101,113,122,131,158],"those":[32],"instances":[33],"its":[36],"current":[37],"prediction":[38],"maximally":[40],"uncertain.":[41],"The":[42,120],"predictions":[43],"as":[44,46,57,104],"well":[45],"measures":[48,90,133],"used":[49],"to":[50,68,92,97,126,138],"quantify":[51],"degree":[53],"uncertainty,":[55,102],"such":[56,103,132],"entropy,":[58],"are":[59],"traditionally":[60],"a":[62,118],"probabilistic":[63],"nature.":[64],"Yet,":[65],"alternative":[66],"approaches":[67],"capturing":[69],"learning,":[73],"alongside":[74],"with":[75,152],"corresponding":[76],"measures,":[78,154],"recent":[83],"years.":[84],"particular,":[86],"some":[87],"these":[89],"seek":[91],"distinguish":[93],"different":[94,99,153],"sources":[95],"and":[96,108,137,164],"separate":[98],"types":[100],"reducible":[106],"(epistemic)":[107],"irreducible":[110],"(aleatoric)":[111],"part":[112],"total":[115],"prediction.":[119],"goal":[121],"this":[123,146],"paper":[124],"elaborate":[127],"on":[128],"usefulness":[130],"compare":[139,165],"their":[140],"performance":[141],"learning.":[144],"To":[145],"end,":[147],"we":[148],"instantiate":[149],"sampling":[151,160],"analyze":[155],"properties":[157],"thus":[162],"obtained,":[163],"them":[166],"an":[168],"experimental":[169],"study.":[170]},"counts_by_year":[{"year":2026,"cited_by_count":39},{"year":2025,"cited_by_count":52},{"year":2024,"cited_by_count":52},{"year":2023,"cited_by_count":33},{"year":2022,"cited_by_count":17},{"year":2021,"cited_by_count":1}],"updated_date":"2026-08-12T21:12:35.861297","created_date":"2025-10-10T00:00:00"}