{"id":"https://openalex.org/W4372346369","doi":"https://doi.org/10.1109/icassp49357.2023.10096531","title":"Masked Token Similarity Transfer for Compressing Transformer-Based ASR Models","display_name":"Masked Token Similarity Transfer for Compressing Transformer-Based ASR Models","publication_year":2023,"publication_date":"2023-05-05","ids":{"openalex":"https://openalex.org/W4372346369","doi":"https://doi.org/10.1109/icassp49357.2023.10096531"},"language":"en","primary_location":{"id":"doi:10.1109/icassp49357.2023.10096531","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icassp49357.2023.10096531","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020621473","display_name":"Euntae Choi","orcid":null},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Euntae Choi","raw_affiliation_strings":["Seoul National University,Department of Computer Science and Engineering","Department of Computer Science and Engineering, Seoul National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Seoul National University,Department of Computer Science and Engineering","institution_ids":["https://openalex.org/I139264467"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059677906","display_name":"Youshin Lim","orcid":"https://orcid.org/0000-0003-3328-0064"},"institutions":[{"id":"https://openalex.org/I49946491","display_name":"Hyundai Motors (South Korea)","ror":"https://ror.org/016kvft77","country_code":"KR","type":"company","lineage":["https://openalex.org/I197312522","https://openalex.org/I49946491"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Youshin Lim","raw_affiliation_strings":["42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","institution_ids":["https://openalex.org/I49946491"]},{"raw_affiliation_string":"Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I49946491"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053784244","display_name":"Byeong-Yeol Kim","orcid":"https://orcid.org/0000-0001-6019-5047"},"institutions":[{"id":"https://openalex.org/I49946491","display_name":"Hyundai Motors (South Korea)","ror":"https://ror.org/016kvft77","country_code":"KR","type":"company","lineage":["https://openalex.org/I197312522","https://openalex.org/I49946491"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Byeong-Yeol Kim","raw_affiliation_strings":["42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","institution_ids":["https://openalex.org/I49946491"]},{"raw_affiliation_string":"Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I49946491"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031214691","display_name":"Hyung Yong Kim","orcid":null},"institutions":[{"id":"https://openalex.org/I49946491","display_name":"Hyundai Motors (South Korea)","ror":"https://ror.org/016kvft77","country_code":"KR","type":"company","lineage":["https://openalex.org/I197312522","https://openalex.org/I49946491"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hyung Yong Kim","raw_affiliation_strings":["42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","institution_ids":["https://openalex.org/I49946491"]},{"raw_affiliation_string":"Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I49946491"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025565566","display_name":"Hanbin Lee","orcid":"https://orcid.org/0000-0002-4545-0027"},"institutions":[{"id":"https://openalex.org/I49946491","display_name":"Hyundai Motors (South Korea)","ror":"https://ror.org/016kvft77","country_code":"KR","type":"company","lineage":["https://openalex.org/I197312522","https://openalex.org/I49946491"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hanbin Lee","raw_affiliation_strings":["42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","institution_ids":["https://openalex.org/I49946491"]},{"raw_affiliation_string":"Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I49946491"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027173062","display_name":"Yunkyu Lim","orcid":null},"institutions":[{"id":"https://openalex.org/I49946491","display_name":"Hyundai Motors (South Korea)","ror":"https://ror.org/016kvft77","country_code":"KR","type":"company","lineage":["https://openalex.org/I197312522","https://openalex.org/I49946491"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yunkyu Lim","raw_affiliation_strings":["42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","institution_ids":["https://openalex.org/I49946491"]},{"raw_affiliation_string":"Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I49946491"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114048496","display_name":"Seung Woo Yu","orcid":null},"institutions":[{"id":"https://openalex.org/I49946491","display_name":"Hyundai Motors (South Korea)","ror":"https://ror.org/016kvft77","country_code":"KR","type":"company","lineage":["https://openalex.org/I197312522","https://openalex.org/I49946491"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Seung Woo Yu","raw_affiliation_strings":["42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"42dot.ai,Hyundai Motor Company,Seoul,Republic of Korea","institution_ids":["https://openalex.org/I49946491"]},{"raw_affiliation_string":"Hyundai Motor Company, 42dot.ai, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I49946491"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063521444","display_name":"Sungjoo Yoo","orcid":"https://orcid.org/0000-0002-5853-0675"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Sungjoo Yoo","raw_affiliation_strings":["Seoul National University,Department of Computer Science and Engineering","Department of Computer Science and Engineering, Seoul National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Seoul National University,Department of Computer Science and Engineering","institution_ids":["https://openalex.org/I139264467"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1632,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.5270219,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"33","issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.8522965908050537},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7547033429145813},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6525273323059082},{"id":"https://openalex.org/keywords/transfer-of-learning","display_name":"Transfer of learning","score":0.5334572792053223},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.529966413974762},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.49318069219589233},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.48110145330429077},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4751857817173004},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.4721660315990448},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.10651877522468567},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08212405443191528},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.07909929752349854}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.8522965908050537},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7547033429145813},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6525273323059082},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.5334572792053223},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.529966413974762},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.49318069219589233},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.48110145330429077},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4751857817173004},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.4721660315990448},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.10651877522468567},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08212405443191528},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.07909929752349854},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp49357.2023.10096531","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icassp49357.2023.10096531","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1821462560","https://openalex.org/W2127141656","https://openalex.org/W2896457183","https://openalex.org/W2933138175","https://openalex.org/W2936774411","https://openalex.org/W2970454332","https://openalex.org/W2973049979","https://openalex.org/W2975381464","https://openalex.org/W3004127093","https://openalex.org/W3005680577","https://openalex.org/W3035060554","https://openalex.org/W3036601975","https://openalex.org/W3197580070","https://openalex.org/W3203140070","https://openalex.org/W3209059054","https://openalex.org/W4225529283","https://openalex.org/W4297841337","https://openalex.org/W4297841557","https://openalex.org/W4312268073","https://openalex.org/W4385245566","https://openalex.org/W6631190155","https://openalex.org/W6638523607","https://openalex.org/W6739901393","https://openalex.org/W6755207826","https://openalex.org/W6768080748","https://openalex.org/W6774314701","https://openalex.org/W6779326418","https://openalex.org/W6780218876","https://openalex.org/W6780226713"],"related_works":["https://openalex.org/W3016124757","https://openalex.org/W3049463507","https://openalex.org/W2936497627","https://openalex.org/W4288365749","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W4287598411","https://openalex.org/W3098382480","https://openalex.org/W3198458223","https://openalex.org/W4288267738"],"abstract_inverted_index":{"Recent":[0],"self-supervised":[1],"automatic":[2],"speech":[3],"recognition":[4],"(ASR)":[5],"models":[6],"based":[7],"on":[8,23,138],"transformers":[9],"are":[10],"showing":[11],"best":[12],"performance,":[13],"but":[14],"their":[15],"footprint":[16],"is":[17,104,144],"too":[18],"large":[19],"to":[20,28,37,51,57,130],"be":[21,35,154],"trained":[22],"low-resource":[24],"environments":[25],"or":[26],"deployed":[27],"edge":[29],"devices.":[30],"Knowledge":[31],"distillation":[32],"(KD)":[33],"can":[34],"employed":[36],"reduce":[38,124],"the":[39,78,93,100,125],"model":[40,126],"size.":[41],"However,":[42],"setting":[43,116],"embedding":[44],"dimension":[45],"of":[46,81,128,132],"teacher":[47,82],"and":[48,99],"student":[49,76,129],"network":[50],"different":[52],"values":[53],"makes":[54],"it":[55],"difficult":[56],"transfer":[58,89,114],"token":[59,87,98,109],"embeddings":[60],"for":[61],"better":[62],"performance.":[63],"To":[64],"mitigate":[65],"this":[66],"issue,":[67],"we":[68],"present":[69],"a":[70,97,107,118],"novel":[71],"KD":[72],"method":[73],"in":[74,141],"which":[75,146],"mimics":[77],"prediction":[79],"vector":[80],"under":[83],"our":[84,113,121],"proposed":[85,122],"masked":[86],"similarity":[88,110],"(MTST)":[90],"loss":[91],"where":[92],"temporal":[94],"relation":[95],"between":[96],"other":[101],"unmasked":[102],"ones":[103],"encoded":[105],"into":[106],"dimension-agnostic":[108],"vector.":[111],"Under":[112],"learning":[115],"with":[117],"fine-tuned":[119],"teacher,":[120],"methods":[123],"size":[127],"28.3%":[131],"teacher\u2019s":[133],"while":[134],"word":[135],"error":[136],"rate":[137],"test-clean":[139],"subset":[140],"LibriSpeech":[142],"corpus":[143],"4.93%,":[145],"surpasses":[147],"prior":[148],"works.":[149],"Our":[150],"source":[151],"code":[152],"will":[153],"made":[155],"available.":[156]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}