{"id":"https://openalex.org/W3000462134","doi":"https://doi.org/10.48550/arxiv.2001.04694","title":"Hydra: Preserving Ensemble Diversity for Model Distillation","display_name":"Hydra: Preserving Ensemble Diversity for Model Distillation","alternate_titles":[],"publication_year":2020,"publication_date":"2020-01-14","ids":{"openalex":"https://openalex.org/W3000462134","doi":"https://doi.org/10.48550/arxiv.2001.04694","mag":"3000462134"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2001.04694","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.04694","pdf_url":"https://arxiv.org/pdf/2001.04694","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W3000462134"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2001.04694","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Tran, Linh","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Tran, Linh","raw_affiliation_strings":["Imperial College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Imperial College London","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022871230","display_name":"Bastiaan S. Veeling","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Veeling, Bastiaan S.","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Roth, Kevin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Roth, Kevin","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065197523","display_name":"Jakub \u015awi\u0105tkowski","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Swiatkowski, Jakub","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021577029","display_name":"Joshua V. Dillon","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Dillon, Joshua V.","raw_affiliation_strings":["Google Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112347601","display_name":"Jasper Snoek","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Snoek, Jasper","raw_affiliation_strings":["Google Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036302820","display_name":"Stephan Mandt","orcid":"https://orcid.org/0000-0001-7836-7839","observed_orcids":["https://orcid.org/0000-0001-7836-7839"]},"institutions":[{"id":"https://openalex.org/I204250578","display_name":"University of California, Irvine","ror":"https://ror.org/04gyf1771","country_code":"US","type":"education","lineage":["https://openalex.org/I204250578"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Mandt, Stephan","raw_affiliation_strings":["University of California , Irvine"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Irvine","institution_ids":["https://openalex.org/I204250578"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082512329","display_name":"Tim Salimans","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Salimans, Tim","raw_affiliation_strings":["Google Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000291599","display_name":"Sebastian Nowozin","orcid":"https://orcid.org/0000-0003-0347-0852","observed_orcids":["https://orcid.org/0000-0003-0347-0852"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Nowozin, Sebastian","raw_affiliation_strings":["Google Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033851429","display_name":"Rodolphe Jenatton","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jenatton, Rodolphe","raw_affiliation_strings":["Google Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":74,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.6923999786376953,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.6923999786376953,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.12470000237226486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.11190000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-distillation","display_name":"model distillation","score":0.8230000138282776},{"id":"https://openalex.org/keywords/ensemble-diversity","display_name":"ensemble diversity","score":0.6589999794960022},{"id":"https://openalex.org/keywords/uncertainty-estimation","display_name":"uncertainty estimation","score":0.7009999752044678},{"id":"https://openalex.org/keywords/ensemble-distillation","display_name":"ensemble distillation","score":0.4490000009536743},{"id":"https://openalex.org/keywords/out-of-distribution-detection","display_name":"out-of-distribution detection","score":0.753000020980835},{"id":"https://openalex.org/keywords/classification","display_name":"classification","score":0.4790000021457672},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"regression analysis","score":0.9509999752044678}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7281206250190735},{"id":"https://openalex.org/C119898033","wikidata":"https://www.wikidata.org/wiki/Q3433888","display_name":"Ensemble forecasting","level":2,"score":0.677078127861023},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6326605677604675},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.5331700444221497},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.531150221824646},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5228638648986816},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5222302675247192},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.464780330657959},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.46337902545928955},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.45965972542762756},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2224976122379303},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2001.04694","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.04694","pdf_url":"https://arxiv.org/pdf/2001.04694","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2001.04694","doi":"https://doi.org/10.48550/arxiv.2001.04694","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2001.04694","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2001.04694","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.04694","pdf_url":"https://arxiv.org/pdf/2001.04694","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3000462134.pdf","grobid_xml":"https://content.openalex.org/works/W3000462134.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W1534477342","https://openalex.org/W1592410721","https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W2025720061","https://openalex.org/W2097117768","https://openalex.org/W2134797427","https://openalex.org/W2158471180","https://openalex.org/W2198724430","https://openalex.org/W2253986341","https://openalex.org/W2257113116","https://openalex.org/W2280163991","https://openalex.org/W2294370754","https://openalex.org/W2560321925","https://openalex.org/W2620998106","https://openalex.org/W2765867474","https://openalex.org/W2787017828","https://openalex.org/W2807912816","https://openalex.org/W2945289329","https://openalex.org/W2948194985","https://openalex.org/W2963215553","https://openalex.org/W2963373431","https://openalex.org/W2973651390","https://openalex.org/W2996184071","https://openalex.org/W3120740533"],"related_works":["https://openalex.org/W3007318993","https://openalex.org/W2996184071","https://openalex.org/W3209390122","https://openalex.org/W7133242094","https://openalex.org/W2944942767","https://openalex.org/W4226156885","https://openalex.org/W7010728218","https://openalex.org/W4281263877","https://openalex.org/W3161663089","https://openalex.org/W4283779092"],"abstract_inverted_index":{"Ensembles":[0],"of":[1,16,45,72,90,95,105,144,172],"models":[2],"have":[3],"been":[4],"empirically":[5],"shown":[6],"to":[7,12,50,93,123,139],"improve":[8],"predictive":[9,53,142],"performance":[10,161],"and":[11,24,42,164,179],"yield":[13],"robust":[14],"measures":[15],"uncertainty.":[17],"However,":[18],"they":[19],"are":[20],"expensive":[21],"in":[22,155],"computation":[23],"memory.":[25],"Therefore,":[26],"recent":[27],"research":[28],"has":[29],"focused":[30],"on":[31,114,162],"distilling":[32],"ensembles":[33],"into":[34],"a":[35,68,88,110,115,131,152],"single":[36,116],"compact":[37],"model,":[38],"reducing":[39],"the":[40,46,60,70,73,83,96,103,106,141,169,173],"computational":[41],"memory":[43],"burden":[44],"ensemble":[47,61,74,146,175],"while":[48,167],"trying":[49],"preserve":[51],"its":[52,64],"behavior.":[54],"Most":[55],"existing":[56],"distillation":[57,111,160],"formulations":[58],"summarize":[59],"by":[62],"capturing":[63,168],"average":[65],"predictions.":[66],"As":[67],"result,":[69],"diversity":[71,104],"predictions,":[75],"stemming":[76],"from":[77],"each":[78,137,145],"member,":[79],"is":[80],"lost.":[81],"Thus,":[82],"distilled":[84],"model":[85],"cannot":[86],"provide":[87],"measure":[89],"uncertainty":[91,170],"comparable":[92],"that":[94,135,150],"original":[97,174],"ensemble.":[98],"To":[99],"retain":[100],"more":[101],"faithfully":[102],"ensemble,":[107],"we":[108,121],"propose":[109],"method":[112],"based":[113],"multi-headed":[117],"neural":[118],"network,":[119],"which":[120],"refer":[122],"as":[124],"Hydra.":[125],"The":[126],"shared":[127],"body":[128],"network":[129],"learns":[130],"joint":[132],"feature":[133],"representation":[134],"enables":[136],"head":[138],"capture":[140],"behavior":[143,171],"member.":[147],"We":[148],"demonstrate":[149],"with":[151],"slight":[153],"increase":[154],"parameter":[156],"count,":[157],"Hydra":[158],"improves":[159],"classification":[163],"regression":[165],"settings":[166],"over":[176],"both":[177],"in-domain":[178],"out-of-distribution":[180],"tasks.":[181]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3000462134","counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":20},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":8}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2020-01-23T00:00:00"}
