{"id":"https://openalex.org/W3160799772","doi":"https://doi.org/10.1109/icassp39728.2021.9414460","title":"Hubert: How Much Can a Bad Teacher Benefit ASR Pre-Training?","display_name":"Hubert: How Much Can a Bad Teacher Benefit ASR Pre-Training?","publication_year":2021,"publication_date":"2021-05-13","ids":{"openalex":"https://openalex.org/W3160799772","doi":"https://doi.org/10.1109/icassp39728.2021.9414460","mag":"3160799772"},"language":"en","primary_location":{"id":"doi:10.1109/icassp39728.2021.9414460","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp39728.2021.9414460","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051950818","display_name":"Wei-Ning Hsu","orcid":"https://orcid.org/0000-0001-5546-5217"},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Wei-Ning Hsu","raw_affiliation_strings":["Facebook AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI Research","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015826285","display_name":"Yao-Hung Hubert Tsai","orcid":"https://orcid.org/0000-0001-5312-1875"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yao-Hung Hubert Tsai","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113811530","display_name":"Benjamin Bolte","orcid":null},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Benjamin Bolte","raw_affiliation_strings":["Facebook AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI Research","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071983998","display_name":"Ruslan Salakhutdinov","orcid":"https://orcid.org/0000-0002-3752-2756"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruslan Salakhutdinov","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103742478","display_name":"Abdelrahman Mohamed","orcid":null},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Abdelrahman Mohamed","raw_affiliation_strings":["Facebook AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI Research","institution_ids":["https://openalex.org/I2252078561"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":116,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"6533","last_page":"6537"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7828772068023682},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.6961578726768494},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6453984379768372},{"id":"https://openalex.org/keywords/utterance","display_name":"Utterance","score":0.6206941604614258},{"id":"https://openalex.org/keywords/lexicon","display_name":"Lexicon","score":0.5961989760398865},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5215259194374084},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.5164957046508789},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4908643364906311},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.48415929079055786},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.46147796511650085},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4459178149700165},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4310789108276367},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.42941582202911377},{"id":"https://openalex.org/keywords/variable","display_name":"Variable (mathematics)","score":0.42527514696121216},{"id":"https://openalex.org/keywords/acoustic-model","display_name":"Acoustic model","score":0.41943174600601196},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.41881832480430603},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11441567540168762},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.08870983123779297}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7828772068023682},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.6961578726768494},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6453984379768372},{"id":"https://openalex.org/C2775852435","wikidata":"https://www.wikidata.org/wiki/Q258403","display_name":"Utterance","level":2,"score":0.6206941604614258},{"id":"https://openalex.org/C2778121359","wikidata":"https://www.wikidata.org/wiki/Q8096","display_name":"Lexicon","level":2,"score":0.5961989760398865},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5215259194374084},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.5164957046508789},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4908643364906311},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.48415929079055786},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.46147796511650085},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4459178149700165},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4310789108276367},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.42941582202911377},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.42527514696121216},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.41943174600601196},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41881832480430603},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11441567540168762},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.08870983123779297},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp39728.2021.9414460","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp39728.2021.9414460","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8899999856948853,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":74,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1522301498","https://openalex.org/W2100768664","https://openalex.org/W2101234009","https://openalex.org/W2110073835","https://openalex.org/W2127141656","https://openalex.org/W2146444479","https://openalex.org/W2750248772","https://openalex.org/W2752796333","https://openalex.org/W2758785877","https://openalex.org/W2842511635","https://openalex.org/W2883725317","https://openalex.org/W2896457183","https://openalex.org/W2933138175","https://openalex.org/W2946417913","https://openalex.org/W2950180292","https://openalex.org/W2953190524","https://openalex.org/W2962739339","https://openalex.org/W2962850167","https://openalex.org/W2963341956","https://openalex.org/W2963618559","https://openalex.org/W2963799213","https://openalex.org/W2964121744","https://openalex.org/W2970971581","https://openalex.org/W2972943112","https://openalex.org/W2973049979","https://openalex.org/W2979476256","https://openalex.org/W2981283774","https://openalex.org/W2982223350","https://openalex.org/W2988736778","https://openalex.org/W2995181338","https://openalex.org/W2996383576","https://openalex.org/W2998532468","https://openalex.org/W3003875258","https://openalex.org/W3008525923","https://openalex.org/W3011411500","https://openalex.org/W3013571468","https://openalex.org/W3015265920","https://openalex.org/W3015522062","https://openalex.org/W3016011332","https://openalex.org/W3025035610","https://openalex.org/W3033038061","https://openalex.org/W3035202887","https://openalex.org/W3035524453","https://openalex.org/W3036601975","https://openalex.org/W3099782249","https://openalex.org/W3100270690","https://openalex.org/W3125709657","https://openalex.org/W3126816608","https://openalex.org/W3144810982","https://openalex.org/W3148040514","https://openalex.org/W4254197176","https://openalex.org/W4287824654","https://openalex.org/W4295312788","https://openalex.org/W4297786395","https://openalex.org/W4297808394","https://openalex.org/W6631190155","https://openalex.org/W6675022971","https://openalex.org/W6675354045","https://openalex.org/W6745117592","https://openalex.org/W6753000030","https://openalex.org/W6755207826","https://openalex.org/W6760822226","https://openalex.org/W6766978945","https://openalex.org/W6769196770","https://openalex.org/W6769455919","https://openalex.org/W6770514103","https://openalex.org/W6771917389","https://openalex.org/W6772883055","https://openalex.org/W6777232839","https://openalex.org/W6780218876","https://openalex.org/W6780483730","https://openalex.org/W6783797576","https://openalex.org/W6947929050"],"related_works":["https://openalex.org/W2126322296","https://openalex.org/W2955724459","https://openalex.org/W2163537793","https://openalex.org/W2037769829","https://openalex.org/W2916997151","https://openalex.org/W2781555308","https://openalex.org/W2402899696","https://openalex.org/W3021690593","https://openalex.org/W2125343999","https://openalex.org/W4200200210"],"abstract_inverted_index":{"Compared":[0],"to":[1,66,98],"vision":[2],"and":[3,38,121],"language":[4],"applications,":[5],"self-supervised":[6],"pre-training":[7,72,96],"approaches":[8],"for":[9,71],"ASR":[10],"are":[11,19],"challenged":[12],"by":[13],"three":[14],"unique":[15],"problems:":[16],"(1)":[17],"There":[18],"multiple":[20],"sound":[21,36],"units":[22,41],"in":[23],"each":[24],"input":[25],"utterance,":[26],"(2)":[27],"With":[28],"audio-only":[29],"pre-training,":[30],"there":[31],"is":[32,83],"no":[33,46],"lexicon":[34],"of":[35,73,80,103,118,125],"units,":[37],"(3)":[39],"Sound":[40],"have":[42],"variable":[43],"lengths":[44],"with":[45,113],"explicit":[47],"segmentation.":[48],"In":[49],"this":[50],"paper,":[51],"we":[52],"propose":[53],"the":[54,85,89,95,101,104,127,131,137],"Hidden-Unit":[55],"BERT":[56,75],"(HUBERT)":[57],"model":[58,129],"which":[59],"utilizes":[60],"a":[61,74,114],"cheap":[62],"k-means":[63,116],"clustering":[64],"step":[65],"provide":[67],"aligned":[68],"target":[69],"labels":[70],"model.":[76],"A":[77],"key":[78],"ingredient":[79],"our":[81],"approach":[82],"applying":[84],"predictive":[86],"loss":[87],"over":[88],"masked":[90],"regions":[91],"only.":[92],"This":[93],"allows":[94],"stage":[97],"benefit":[99],"from":[100],"consistency":[102],"unsupervised":[105],"teacher":[106,117],"rather":[107],"that":[108],"its":[109],"intrinsic":[110],"quality.":[111],"Starting":[112],"simple":[115],"100":[119],"cluster,":[120],"using":[122],"two":[123],"iterations":[124],"clustering,":[126],"HUBERT":[128],"matches":[130],"state-of-the-art":[132],"wav2vec":[133],"2.0":[134],"performance":[135],"on":[136],"ultra":[138],"low-resource":[139],"Libri-light":[140],"10h,":[141],"1h,":[142],"10min":[143],"supervised":[144],"subsets.":[145]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":28},{"year":2022,"cited_by_count":41},{"year":2021,"cited_by_count":14}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}