{"id":"https://openalex.org/W2995464762","doi":"https://doi.org/10.48550/arxiv.2002.06470","title":"Pitfalls of In-Domain Uncertainty Estimation and Ensembling in Deep Learning","display_name":"Pitfalls of In-Domain Uncertainty Estimation and Ensembling in Deep Learning","publication_year":2020,"publication_date":"2020-02-15","ids":{"openalex":"https://openalex.org/W2995464762","doi":"https://doi.org/10.48550/arxiv.2002.06470","mag":"2995464762"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2002.06470","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.06470","pdf_url":"https://arxiv.org/pdf/2002.06470","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2002.06470","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008914065","display_name":"Arsenii Ashukha","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I118501908","display_name":"National Research University Higher School of Economics","ror":"https://ror.org/055f7t516","country_code":"RU","type":"education","lineage":["https://openalex.org/I118501908"]},{"id":"https://openalex.org/I4210141363","display_name":"Samsung (Russia)","ror":"https://ror.org/051an6p98","country_code":"RU","type":"company","lineage":["https://openalex.org/I4210141363"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Ashukha, Arsenii","raw_affiliation_strings":["Samsung AI Center Moscow","National research university Higher School of Economics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center Moscow","institution_ids":["https://openalex.org/I4210141363"]},{"raw_affiliation_string":"National research university Higher School of Economics","institution_ids":["https://openalex.org/I118501908"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077626187","display_name":"Alexander Lyzhov","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I118501908","display_name":"National Research University Higher School of Economics","ror":"https://ror.org/055f7t516","country_code":"RU","type":"education","lineage":["https://openalex.org/I118501908"]},{"id":"https://openalex.org/I125989756","display_name":"Skolkovo Institute of Science and Technology","ror":"https://ror.org/03f9nc143","country_code":"RU","type":"education","lineage":["https://openalex.org/I125989756"]},{"id":"https://openalex.org/I4210141363","display_name":"Samsung (Russia)","ror":"https://ror.org/051an6p98","country_code":"RU","type":"company","lineage":["https://openalex.org/I4210141363"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Lyzhov, Alexander","raw_affiliation_strings":["National Research University Higher School of Economics Skoltech refers to Skolkovo Institute of Science and Technology","Samsung AI Center Moscow , Skoltech","National research university Higher School of Economics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Research University Higher School of Economics Skoltech refers to Skolkovo Institute of Science and Technology","institution_ids":["https://openalex.org/I118501908","https://openalex.org/I125989756"]},{"raw_affiliation_string":"Samsung AI Center Moscow , Skoltech","institution_ids":["https://openalex.org/I125989756","https://openalex.org/I4210141363"]},{"raw_affiliation_string":"National research university Higher School of Economics","institution_ids":["https://openalex.org/I118501908"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087697434","display_name":"Dmitry Molchanov","orcid":"https://orcid.org/0000-0002-4197-5029","observed_orcids":["https://orcid.org/0000-0002-4197-5029"]},"institutions":[{"id":"https://openalex.org/I118501908","display_name":"National Research University Higher School of Economics","ror":"https://ror.org/055f7t516","country_code":"RU","type":"education","lineage":["https://openalex.org/I118501908"]},{"id":"https://openalex.org/I4210141363","display_name":"Samsung (Russia)","ror":"https://ror.org/051an6p98","country_code":"RU","type":"company","lineage":["https://openalex.org/I4210141363"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Molchanov, Dmitry","raw_affiliation_strings":["Samsung AI Center Moscow","National research university Higher School of Economics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center Moscow","institution_ids":["https://openalex.org/I4210141363"]},{"raw_affiliation_string":"National research university Higher School of Economics","institution_ids":["https://openalex.org/I118501908"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101595563","display_name":"Dmitry Vetrov","orcid":"https://orcid.org/0000-0001-6863-9028","observed_orcids":["https://orcid.org/0000-0001-6863-9028","https://orcid.org/0000-0001-6779-6283"]},"institutions":[{"id":"https://openalex.org/I118501908","display_name":"National Research University Higher School of Economics","ror":"https://ror.org/055f7t516","country_code":"RU","type":"education","lineage":["https://openalex.org/I118501908"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Vetrov, Dmitry","raw_affiliation_strings":["National research university Higher School of Economics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National research university Higher School of Economics","institution_ids":["https://openalex.org/I118501908"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":275,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"29"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.7378000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.7378000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.13429999351501465,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.08479999750852585,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.8140000104904175},{"id":"https://openalex.org/keywords/uncertainty-measures","display_name":"uncertainty measures","score":0.8539999723434448},{"id":"https://openalex.org/keywords/ensemble-equivalence","display_name":"ensemble equivalence","score":0.6050000190734863},{"id":"https://openalex.org/keywords/uncertainty-quantification","display_name":"uncertainty quantification","score":0.3880000114440918}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.720585823059082},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7079145312309265},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.7054712772369385},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6474038362503052},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5639293789863586},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.548860490322113},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.506508469581604},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.4952599108219147},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.47118639945983887},{"id":"https://openalex.org/C137209882","wikidata":"https://www.wikidata.org/wiki/Q1403517","display_name":"Measurement uncertainty","level":2,"score":0.4628749191761017},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.44280073046684265},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3550962805747986},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.12348726391792297},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11305472254753113},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07585620880126953},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2002.06470","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.06470","pdf_url":"https://arxiv.org/pdf/2002.06470","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2002.06470","doi":"https://doi.org/10.48550/arxiv.2002.06470","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2002.06470","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"},{"id":"mag:2995464762","doi":null,"is_oa":false,"landing_page_url":"https://www.openreview.net/pdf?id=BJxI5gHKDr","pdf_url":null,"source":{"id":"https://openalex.org/S4306419637","display_name":"International Conference on Learning Representations","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Learning Representations","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2002.06470","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.06470","pdf_url":"https://arxiv.org/pdf/2002.06470","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1476772921","display_name":null,"funder_award_id":"19-71-30020","funder_id":"https://openalex.org/F4320324099","funder_display_name":"Russian Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320324099","display_name":"Russian Science Foundation","ror":"https://ror.org/03y2gwe85"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2995464762.pdf","grobid_xml":"https://content.openalex.org/works/W2995464762.grobid-xml"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W35527955","https://openalex.org/W582134693","https://openalex.org/W1592410721","https://openalex.org/W1673923490","https://openalex.org/W1686810756","https://openalex.org/W1821462560","https://openalex.org/W1826234144","https://openalex.org/W1836465849","https://openalex.org/W2073241381","https://openalex.org/W2097117768","https://openalex.org/W2117539524","https://openalex.org/W2135293965","https://openalex.org/W2155894447","https://openalex.org/W2158471180","https://openalex.org/W2167433878","https://openalex.org/W2194775991","https://openalex.org/W2254249950","https://openalex.org/W2269892441","https://openalex.org/W2560321925","https://openalex.org/W2582745083","https://openalex.org/W2588860837","https://openalex.org/W2592505114","https://openalex.org/W2612983688","https://openalex.org/W2626967530","https://openalex.org/W2749581528","https://openalex.org/W2765407302","https://openalex.org/W2785783085","https://openalex.org/W2786857698","https://openalex.org/W2787825417","https://openalex.org/W2792287754","https://openalex.org/W2809702612","https://openalex.org/W2894954915","https://openalex.org/W2899771611","https://openalex.org/W2907020378","https://openalex.org/W2912168444","https://openalex.org/W2914331073","https://openalex.org/W2917792182","https://openalex.org/W2919841361","https://openalex.org/W2921127884","https://openalex.org/W2947023772","https://openalex.org/W2948194985","https://openalex.org/W2948210138","https://openalex.org/W2949117887","https://openalex.org/W2951266961","https://openalex.org/W2952746978","https://openalex.org/W2963215553","https://openalex.org/W2963384892","https://openalex.org/W2963960193","https://openalex.org/W2964137095","https://openalex.org/W2964153729","https://openalex.org/W2969338701","https://openalex.org/W2990037398","https://openalex.org/W3000462134","https://openalex.org/W3021901740","https://openalex.org/W3034403194","https://openalex.org/W3118608800"],"related_works":["https://openalex.org/W3035136782","https://openalex.org/W2980854601","https://openalex.org/W4306808678","https://openalex.org/W6944539299","https://openalex.org/W4316036014","https://openalex.org/W7126201310","https://openalex.org/W4226156885","https://openalex.org/W3154270017","https://openalex.org/W4223925428","https://openalex.org/W4313491975"],"abstract_inverted_index":{"Uncertainty":[0,7],"estimation":[1,8],"and":[2,52,86],"ensembling":[3,18,69,91],"methods":[4],"go":[5],"hand-in-hand.":[6],"is":[9],"one":[10],"of":[11,17,56,67,98,106],"the":[12,21,47,80],"main":[13],"benchmarks":[14],"for":[15,42,49],"assessment":[16],"performance.":[19,108],"At":[20],"same":[22],"time,":[23],"deep":[24,81],"learning":[25],"ensembles":[26],"have":[27],"provided":[28],"state-of-the-art":[29],"results":[30],"in":[31,75,104],"uncertainty":[32,41],"estimation.":[33],"In":[34],"this":[35,76],"work,":[36],"we":[37,62,78],"focus":[38],"on":[39],"in-domain":[40],"image":[43],"classification.":[44],"We":[45],"explore":[46],"standards":[48],"its":[50],"quantification":[51],"point":[53],"out":[54],"pitfalls":[55],"existing":[57],"metrics.":[58],"Avoiding":[59],"these":[60],"pitfalls,":[61],"perform":[63],"a":[64],"broad":[65],"study":[66],"different":[68],"techniques.":[70],"To":[71],"provide":[72],"more":[73],"insight":[74],"study,":[77],"introduce":[79],"ensemble":[82,97],"equivalent":[83,94],"score":[84],"(DEE)":[85],"show":[87],"that":[88],"many":[89],"sophisticated":[90],"techniques":[92],"are":[93],"to":[95],"an":[96],"only":[99],"few":[100],"independently":[101],"trained":[102],"networks":[103],"terms":[105],"test":[107]},"counts_by_year":[{"year":2026,"cited_by_count":24},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":44},{"year":2023,"cited_by_count":52},{"year":2022,"cited_by_count":46},{"year":2021,"cited_by_count":49},{"year":2020,"cited_by_count":37}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2019-12-26T00:00:00"}