{"id":"https://openalex.org/W4392910324","doi":"https://doi.org/10.1109/icassp48485.2024.10445949","title":"Exploiting Audio-Visual Features with Pretrained AV-HuBERT for Multi-Modal Dysarthric Speech Reconstruction","display_name":"Exploiting Audio-Visual Features with Pretrained AV-HuBERT for Multi-Modal Dysarthric Speech Reconstruction","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392910324","doi":"https://doi.org/10.1109/icassp48485.2024.10445949"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10445949","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10445949","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108049724","display_name":"Xueyuan Chen","orcid":"https://orcid.org/0000-0003-0493-839X"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Xueyuan Chen","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","Tsinghua-CUHK Joint Research Center for Media Sciences, Technologies and Systems, Shenzhen International Graduate School, Tsinghua University, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Tsinghua-CUHK Joint Research Center for Media Sciences, Technologies and Systems, Shenzhen International Graduate School, Tsinghua University, Shenzhen, China","institution_ids":["https://openalex.org/I889458895","https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101632921","display_name":"Yuejiao Wang","orcid":"https://orcid.org/0000-0001-5910-9439"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yuejiao Wang","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025119324","display_name":"Xixin Wu","orcid":"https://orcid.org/0000-0001-9543-1572"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xixin Wu","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","Vocal Engineering Technologies Limited, Hong Kong SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Vocal Engineering Technologies Limited, Hong Kong SAR, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068726270","display_name":"Disong Wang","orcid":"https://orcid.org/0000-0001-8481-6880"},"institutions":[{"id":"https://openalex.org/I4210156337","display_name":"Voci (United States)","ror":"https://ror.org/05tdgz517","country_code":"US","type":"company","lineage":["https://openalex.org/I4210156337"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Disong Wang","raw_affiliation_strings":["Vocal Engineering Technologies Limited,Hong Kong SAR,China","Vocal Engineering Technologies Limited, Hong Kong SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Vocal Engineering Technologies Limited,Hong Kong SAR,China","institution_ids":["https://openalex.org/I4210156337"]},{"raw_affiliation_string":"Vocal Engineering Technologies Limited, Hong Kong SAR, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102869280","display_name":"Zhiyong Wu","orcid":"https://orcid.org/0000-0001-8533-0524"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Zhiyong Wu","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","Tsinghua-CUHK Joint Research Center for Media Sciences, Technologies and Systems, Shenzhen International Graduate School, Tsinghua University, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Tsinghua-CUHK Joint Research Center for Media Sciences, Technologies and Systems, Shenzhen International Graduate School, Tsinghua University, Shenzhen, China","institution_ids":["https://openalex.org/I889458895","https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037109470","display_name":"Xunying Liu","orcid":"https://orcid.org/0000-0001-6725-1160"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xunying Liu","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019458385","display_name":"Helen Meng","orcid":"https://orcid.org/0000-0002-4427-3532"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Helen Meng","raw_affiliation_strings":["The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","Tsinghua-CUHK Joint Research Center for Media Sciences, Technologies and Systems, Shenzhen International Graduate School, Tsinghua University, Shenzhen, China","Vocal Engineering Technologies Limited, Hong Kong SAR, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong,Department of Systems Engineering and Engineering Management,Hong Kong SAR,China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Systems Engineering and Engineering Management, The Chinese University of Hong Kong, Hong Kong SAR, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Tsinghua-CUHK Joint Research Center for Media Sciences, Technologies and Systems, Shenzhen International Graduate School, Tsinghua University, Shenzhen, China","institution_ids":["https://openalex.org/I889458895","https://openalex.org/I99065089"]},{"raw_affiliation_string":"Vocal Engineering Technologies Limited, Hong Kong SAR, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.8448,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.91944085,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"12341","last_page":"12345"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9857000112533569,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10863","display_name":"Voice and Speech Disorders","score":0.9855999946594238,"subfield":{"id":"https://openalex.org/subfields/2737","display_name":"Physiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7237399816513062},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7094014883041382},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6621379852294922},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.5087886452674866},{"id":"https://openalex.org/keywords/dysarthria","display_name":"Dysarthria","score":0.4191679358482361},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37017449736595154},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3548191785812378},{"id":"https://openalex.org/keywords/audiology","display_name":"Audiology","score":0.11259227991104126},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.08834466338157654}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7237399816513062},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7094014883041382},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6621379852294922},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.5087886452674866},{"id":"https://openalex.org/C2777639682","wikidata":"https://www.wikidata.org/wiki/Q225957","display_name":"Dysarthria","level":2,"score":0.4191679358482361},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37017449736595154},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3548191785812378},{"id":"https://openalex.org/C548259974","wikidata":"https://www.wikidata.org/wiki/Q569965","display_name":"Audiology","level":1,"score":0.11259227991104126},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.08834466338157654},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10445949","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10445949","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2603945996","display_name":null,"funder_award_id":"62076144","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322942","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W180052447","https://openalex.org/W1522301498","https://openalex.org/W1686810756","https://openalex.org/W2005768155","https://openalex.org/W2121973264","https://openalex.org/W2142300631","https://openalex.org/W2171360373","https://openalex.org/W2527729766","https://openalex.org/W2547364378","https://openalex.org/W2562848127","https://openalex.org/W2748881396","https://openalex.org/W2890952074","https://openalex.org/W2935794029","https://openalex.org/W2949662773","https://openalex.org/W2962788625","https://openalex.org/W2972413216","https://openalex.org/W3015338123","https://openalex.org/W3015430779","https://openalex.org/W3015707856","https://openalex.org/W3033411150","https://openalex.org/W3104792420","https://openalex.org/W3196305160","https://openalex.org/W4221153068","https://openalex.org/W4248797753","https://openalex.org/W4392903361","https://openalex.org/W6727697161","https://openalex.org/W6776960109","https://openalex.org/W6847363464","https://openalex.org/W6849382973","https://openalex.org/W6917585676"],"related_works":["https://openalex.org/W2331173358","https://openalex.org/W2969484279","https://openalex.org/W3160456149","https://openalex.org/W2168872498","https://openalex.org/W4239608382","https://openalex.org/W2329762060","https://openalex.org/W2517018229","https://openalex.org/W2351669973","https://openalex.org/W2312964388","https://openalex.org/W2980869992"],"abstract_inverted_index":{"Dysarthric":[0],"speech":[1,8,11,80,160,179],"reconstruction":[2],"(DSR)":[3],"aims":[4],"to":[5,46,73,89,107,117],"transform":[6],"dysarthric":[7,79,175],"into":[9],"normal":[10,92],"by":[12],"improving":[13],"the":[14,60,91,99,109,119,124,140,166,177],"intelligibility":[15,161],"and":[16,29,95,113,130,135,162],"naturalness.":[17],"This":[18],"is":[19],"a":[20,42,70,86,104,115],"challenging":[21],"task":[22],"especially":[23,164],"for":[24,58,165,187],"patients":[25,188],"with":[26,81,168,173,189],"severe":[27,170,191],"dysarthria":[28,192],"speaking":[30],"in":[31,53,157],"complex,":[32],"noisy":[33],"acoustic":[34],"environments.":[35],"To":[36],"address":[37],"these":[38],"challenges,":[39],"we":[40],"propose":[41],"novel":[43],"multi-modal":[44,65,71],"framework":[45,66],"utilize":[47],"visual":[48,83],"information,":[49],"e.g.,":[50],"lip":[51],"movements,":[52],"DSR":[54],"as":[55],"extra":[56],"clues":[57],"reconstructing":[59],"highly":[61],"abnormal":[62],"pronunciations.":[63],"The":[64],"consists":[67],"of:":[68],"(i)":[69],"encoder":[72,106],"extract":[74],"robust":[75],"phoneme":[76,93,101,126],"embeddings":[77],"from":[78,98],"auxiliary":[82],"features;":[84],"(ii)":[85],"variance":[87],"adaptor":[88],"infer":[90],"duration":[94],"pitch":[96],"contour":[97],"extracted":[100,125],"embeddings;":[102],"(iii)":[103],"speaker":[105,131],"encode":[108],"speaker\u2019s":[110],"voice":[111],"characteristics;":[112],"(iv)":[114],"mel-decoder":[116],"generate":[118],"reconstructed":[120,178],"mel-spectrogram":[121],"based":[122],"on":[123,139],"embeddings,":[127],"prosodic":[128],"features":[129],"embeddings.":[132],"Both":[133],"objective":[134],"subjective":[136],"evaluations":[137],"conducted":[138],"commonly":[141],"used":[142],"UASpeech":[143],"corpus":[144],"show":[145],"that":[146],"our":[147],"proposed":[148],"approach":[149],"can":[150],"achieve":[151],"significant":[152],"improvements":[153],"over":[154],"baseline":[155],"systems":[156],"terms":[158],"of":[159],"naturalness,":[163],"speakers":[167],"more":[169,190],"symptoms.":[171],"Compared":[172],"original":[174],"speech,":[176],"achieves":[180],"42.1%":[181],"absolute":[182],"word":[183],"error":[184],"rate":[185],"reduction":[186],"levels.1":[193]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}