{"id":"https://openalex.org/W4391021546","doi":"https://doi.org/10.1109/asru57964.2023.10389746","title":"Exploring Effective Distillation of Self-Supervised Speech Models for Automatic Speech Recognition","display_name":"Exploring Effective Distillation of Self-Supervised Speech Models for Automatic Speech Recognition","publication_year":2023,"publication_date":"2023-12-16","ids":{"openalex":"https://openalex.org/W4391021546","doi":"https://doi.org/10.1109/asru57964.2023.10389746"},"language":"en","primary_location":{"id":"doi:10.1109/asru57964.2023.10389746","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/asru57964.2023.10389746","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100733455","display_name":"Yujin Wang","orcid":"https://orcid.org/0000-0001-5982-4088"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yujin Wang","raw_affiliation_strings":["Tsinghua University,Department of Electronic Engineering","Department of Electronic Engineering, Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,Department of Electronic Engineering","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Electronic Engineering, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039664887","display_name":"Changli Tang","orcid":"https://orcid.org/0000-0002-6550-1901"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changli Tang","raw_affiliation_strings":["Tsinghua University,Department of Electronic Engineering","Department of Electronic Engineering, Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,Department of Electronic Engineering","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Electronic Engineering, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100438492","display_name":"Ziyang Ma","orcid":"https://orcid.org/0000-0002-0623-9114"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyang Ma","raw_affiliation_strings":["AI Institute X-LANCE Lab Shanghai Jiao Tong University,MoE Key Lab of Artificial Intelligence,Department of Computer Science and Engineering,Shanghai,China","Department of Computer Science and Engineering, MoE Key Lab of Artificial Intelligence, AI Institute X-LANCE Lab Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Institute X-LANCE Lab Shanghai Jiao Tong University,MoE Key Lab of Artificial Intelligence,Department of Computer Science and Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, MoE Key Lab of Artificial Intelligence, AI Institute X-LANCE Lab Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068357345","display_name":"Zhisheng Zheng","orcid":"https://orcid.org/0000-0001-7761-9790"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhisheng Zheng","raw_affiliation_strings":["AI Institute X-LANCE Lab Shanghai Jiao Tong University,MoE Key Lab of Artificial Intelligence,Department of Computer Science and Engineering,Shanghai,China","Department of Computer Science and Engineering, MoE Key Lab of Artificial Intelligence, AI Institute X-LANCE Lab Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Institute X-LANCE Lab Shanghai Jiao Tong University,MoE Key Lab of Artificial Intelligence,Department of Computer Science and Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, MoE Key Lab of Artificial Intelligence, AI Institute X-LANCE Lab Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101293966","display_name":"Xie Chen","orcid":"https://orcid.org/0000-0001-7423-617X"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xie Chen","raw_affiliation_strings":["AI Institute X-LANCE Lab Shanghai Jiao Tong University,MoE Key Lab of Artificial Intelligence,Department of Computer Science and Engineering,Shanghai,China","Peng Cheng Laboratory, Shenzhen, China","Department of Computer Science and Engineering, MoE Key Lab of Artificial Intelligence, AI Institute X-LANCE Lab Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Institute X-LANCE Lab Shanghai Jiao Tong University,MoE Key Lab of Artificial Intelligence,Department of Computer Science and Engineering,Shanghai,China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Peng Cheng Laboratory, Shenzhen, China","institution_ids":["https://openalex.org/I4210136793"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, MoE Key Lab of Artificial Intelligence, AI Institute X-LANCE Lab Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100692904","display_name":"Wei-Qiang Zhang","orcid":"https://orcid.org/0000-0003-3841-1959"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei-Qiang Zhang","raw_affiliation_strings":["Tsinghua University,Department of Electronic Engineering","Department of Electronic Engineering, Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,Department of Electronic Engineering","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Department of Electronic Engineering, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7938462495803833},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.6712391376495361},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5628024339675903},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.528285562992096},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5213590860366821},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5151747465133667},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5141921043395996},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.48284274339675903},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.46692439913749695},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.43684983253479004},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3583660125732422},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.1145126223564148}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7938462495803833},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.6712391376495361},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5628024339675903},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.528285562992096},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5213590860366821},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5151747465133667},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5141921043395996},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.48284274339675903},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.46692439913749695},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.43684983253479004},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3583660125732422},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.1145126223564148},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/asru57964.2023.10389746","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/asru57964.2023.10389746","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1690739335","https://openalex.org/W2402040300","https://openalex.org/W2407386500","https://openalex.org/W2896457183","https://openalex.org/W2933138175","https://openalex.org/W2978017171","https://openalex.org/W2995181338","https://openalex.org/W3034368386","https://openalex.org/W3036601975","https://openalex.org/W3105966348","https://openalex.org/W3138154797","https://openalex.org/W3197580070","https://openalex.org/W3203140070","https://openalex.org/W3203453034","https://openalex.org/W3209059054","https://openalex.org/W3209984917","https://openalex.org/W4221145109","https://openalex.org/W4224821750","https://openalex.org/W4226033575","https://openalex.org/W4226380987","https://openalex.org/W4281492411","https://openalex.org/W4294103325","https://openalex.org/W4296068827","https://openalex.org/W4297841520","https://openalex.org/W4297841557","https://openalex.org/W4313156423","https://openalex.org/W4375868755","https://openalex.org/W4385823092","https://openalex.org/W6752726010","https://openalex.org/W6755207826","https://openalex.org/W6768851824","https://openalex.org/W6780218876","https://openalex.org/W6810007534"],"related_works":["https://openalex.org/W4389116644","https://openalex.org/W2153315159","https://openalex.org/W3103844505","https://openalex.org/W259157601","https://openalex.org/W4205463238","https://openalex.org/W2761785940","https://openalex.org/W1482209366","https://openalex.org/W2110523656","https://openalex.org/W2521627374","https://openalex.org/W2981954115"],"abstract_inverted_index":{"Self-supervised":[0],"learning":[1],"(SSL)":[2],"has":[3,45],"achieved":[4],"great":[5],"success":[6],"in":[7,94,110,133],"speech":[8,68],"processing,":[9],"but":[10],"always":[11],"with":[12],"a":[13,71,86,97,117],"large":[14],"model":[15,77],"size":[16],"to":[17,29,88,104,122,129],"increase":[18],"the":[19,30,38,59,89,106,124],"modeling":[20],"capacity.":[21],"This":[22],"may":[23],"limit":[24],"its":[25],"potential":[26],"applications":[27],"due":[28],"expensive":[31],"computation":[32],"and":[33,119,137],"memory":[34],"costs":[35],"introduced":[36,101],"by":[37],"oversize":[39],"model.":[40],"Compression":[41],"for":[42,66,102],"SSL":[43,64],"models":[44,65],"become":[46],"an":[47],"important":[48],"research":[49],"direction":[50],"of":[51,62,74,83],"practical":[52],"value.":[53],"To":[54],"this":[55],"end,":[56],"we":[57,115],"explore":[58],"effective":[60,120],"distillation":[61,107],"HuBERT-based":[63],"automatic":[67],"recognition.":[69],"First,":[70],"comprehensive":[72],"study":[73],"different":[75],"student":[76],"structures":[78],"is":[79,100],"conducted.":[80],"On":[81],"top":[82],"this,":[84],"as":[85],"supplement":[87],"regression":[90],"loss":[91,99],"widely":[92],"adopted":[93],"previous":[95],"works,":[96],"discriminative":[98],"HuBERT":[103],"enhance":[105],"performance,":[108],"especially":[109],"low-resource":[111],"scenarios.":[112],"In":[113],"addition,":[114],"design":[116],"simple":[118],"algorithm":[121],"distill":[123],"front-end":[125],"input":[126],"from":[127],"waveform":[128],"Fbank":[130],"feature,":[131],"resulting":[132],"17%":[134],"parameter":[135],"reduction":[136],"doubling":[138],"inference":[139],"speed,":[140],"at":[141],"marginal":[142],"performance":[143],"degradation.":[144]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}