{"id":"https://openalex.org/W3203140070","doi":"https://doi.org/10.1109/icassp43922.2022.9747490","title":"Distilhubert: Speech Representation Learning by Layer-Wise Distillation of Hidden-Unit Bert","display_name":"Distilhubert: Speech Representation Learning by Layer-Wise Distillation of Hidden-Unit Bert","publication_year":2022,"publication_date":"2022-04-27","ids":{"openalex":"https://openalex.org/W3203140070","doi":"https://doi.org/10.1109/icassp43922.2022.9747490","mag":"3203140070"},"language":"en","primary_location":{"id":"doi:10.1109/icassp43922.2022.9747490","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747490","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078976109","display_name":"Heng-Jui Chang","orcid":"https://orcid.org/0000-0002-1690-2610"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":true,"raw_author_name":"Heng-Jui Chang","raw_affiliation_strings":["National Taiwan University,College of Electrical Engineering and Computer Science","College of Electrical Engineering and Computer Science, National Taiwan University"],"affiliations":[{"raw_affiliation_string":"National Taiwan University,College of Electrical Engineering and Computer Science","institution_ids":["https://openalex.org/I16733864"]},{"raw_affiliation_string":"College of Electrical Engineering and Computer Science, National Taiwan University","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078898845","display_name":"Shu-Wen Yang","orcid":"https://orcid.org/0000-0001-5503-9410"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Shu-wen Yang","raw_affiliation_strings":["National Taiwan University,College of Electrical Engineering and Computer Science","College of Electrical Engineering and Computer Science, National Taiwan University"],"affiliations":[{"raw_affiliation_string":"National Taiwan University,College of Electrical Engineering and Computer Science","institution_ids":["https://openalex.org/I16733864"]},{"raw_affiliation_string":"College of Electrical Engineering and Computer Science, National Taiwan University","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040508737","display_name":"Hung-yi Lee","orcid":"https://orcid.org/0000-0002-9654-5747"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Hung-yi Lee","raw_affiliation_strings":["National Taiwan University,College of Electrical Engineering and Computer Science","College of Electrical Engineering and Computer Science, National Taiwan University"],"affiliations":[{"raw_affiliation_string":"National Taiwan University,College of Electrical Engineering and Computer Science","institution_ids":["https://openalex.org/I16733864"]},{"raw_affiliation_string":"College of Electrical Engineering and Computer Science, National Taiwan University","institution_ids":["https://openalex.org/I16733864"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5078976109"],"corresponding_institution_ids":["https://openalex.org/I16733864"],"apc_list":null,"apc_paid":null,"fwci":10.4379,"has_fulltext":false,"cited_by_count":101,"citation_normalized_percentile":{"value":0.98886945,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"7087","last_page":"7091"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8144804239273071},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.8036169409751892},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5872440338134766},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5397243499755859},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5077570676803589},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.48437151312828064},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4708786904811859},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4635689854621887},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.44134968519210815},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4314393699169159},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.41352248191833496},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.06927704811096191}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8144804239273071},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.8036169409751892},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5872440338134766},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5397243499755859},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5077570676803589},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.48437151312828064},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4708786904811859},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4635689854621887},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.44134968519210815},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4314393699169159},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.41352248191833496},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.06927704811096191},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp43922.2022.9747490","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp43922.2022.9747490","pdf_url":null,"source":{"id":"https://openalex.org/S4363607702","display_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1821462560","https://openalex.org/W2024490156","https://openalex.org/W2842511635","https://openalex.org/W2933138175","https://openalex.org/W2963242190","https://openalex.org/W2963403868","https://openalex.org/W2972943112","https://openalex.org/W2973049979","https://openalex.org/W2973157397","https://openalex.org/W2978017171","https://openalex.org/W2979476256","https://openalex.org/W2982223350","https://openalex.org/W2988736778","https://openalex.org/W2996383576","https://openalex.org/W3015213852","https://openalex.org/W3015265920","https://openalex.org/W3016181583","https://openalex.org/W3035202887","https://openalex.org/W3036601975","https://openalex.org/W3041561163","https://openalex.org/W3096587983","https://openalex.org/W3097286738","https://openalex.org/W3099782249","https://openalex.org/W3105966348","https://openalex.org/W3112034174","https://openalex.org/W3157923770","https://openalex.org/W3165666670","https://openalex.org/W3169320628","https://openalex.org/W3179803166","https://openalex.org/W3187822143","https://openalex.org/W3189296823","https://openalex.org/W3197411683","https://openalex.org/W3197580070","https://openalex.org/W3198094329","https://openalex.org/W3198858531","https://openalex.org/W3207558756","https://openalex.org/W3209059054","https://openalex.org/W3213925038","https://openalex.org/W4206375145","https://openalex.org/W4226380987","https://openalex.org/W4285719527","https://openalex.org/W4287121455","https://openalex.org/W4287173589","https://openalex.org/W4297808394","https://openalex.org/W4385245566","https://openalex.org/W6638523607","https://openalex.org/W6739901393","https://openalex.org/W6768851824","https://openalex.org/W6769196770","https://openalex.org/W6770514103","https://openalex.org/W6780218876","https://openalex.org/W6786669483","https://openalex.org/W6795952400","https://openalex.org/W6796551075","https://openalex.org/W6798952882","https://openalex.org/W6844194202"],"related_works":["https://openalex.org/W2787993192","https://openalex.org/W2158269427","https://openalex.org/W4381280689","https://openalex.org/W3033859939","https://openalex.org/W2847365777","https://openalex.org/W3128025644","https://openalex.org/W2355048207","https://openalex.org/W3126051647","https://openalex.org/W2750422482","https://openalex.org/W3125827053"],"abstract_inverted_index":{"Self-supervised":[0],"speech":[1,14,24],"representation":[2],"learning":[3,59],"methods":[4],"like":[5],"wav2vec":[6],"2.0":[7],"and":[8,18,37,48,77,94,102],"Hidden-unit":[9],"BERT":[10],"(HuBERT)":[11],"leverage":[12],"unlabeled":[13],"data":[15],"for":[16,22,44,106],"pre-training":[17,39,100],"offer":[19],"good":[20],"representations":[21,64],"numerous":[23],"processing":[25],"tasks.":[26,87],"Despite":[27],"the":[28,97],"success":[29],"of":[30,99],"these":[31],"methods,":[32],"they":[33],"require":[34],"large":[35],"memory":[36],"high":[38],"costs,":[40],"making":[41],"them":[42],"inaccessible":[43],"researchers":[45],"in":[46,84],"academia":[47],"small":[49],"companies.":[50],"Therefore,":[51],"this":[52],"paper":[53],"introduces":[54],"DistilHuBERT,":[55],"a":[56,66],"novel":[57],"multi-task":[58],"framework":[60],"to":[61],"distill":[62],"hidden":[63],"from":[65],"HuBERT":[67],"model":[68],"directly.":[69],"This":[70],"method":[71],"reduces":[72],"HuBERT\u2019s":[73],"size":[74],"by":[75],"75%":[76],"73%":[78],"faster":[79],"while":[80],"retaining":[81],"most":[82],"performance":[83],"ten":[85],"different":[86],"Moreover,":[88],"DistilHuBERT":[89],"required":[90],"little":[91],"training":[92],"time":[93],"data,":[95],"opening":[96],"possibilities":[98],"personal":[101],"on-device":[103],"SSL":[104],"models":[105],"speech.":[107]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":25},{"year":2024,"cited_by_count":36},{"year":2023,"cited_by_count":36},{"year":2022,"cited_by_count":3}],"updated_date":"2026-04-03T22:45:19.894376","created_date":"2025-10-10T00:00:00"}