{"id":"https://openalex.org/W4384303965","doi":"https://doi.org/10.1109/taslp.2023.3294705","title":"Audio-Visual End-to-End Multi-Channel Speech Separation, Dereverberation and Recognition","display_name":"Audio-Visual End-to-End Multi-Channel Speech Separation, Dereverberation and Recognition","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4384303965","doi":"https://doi.org/10.1109/taslp.2023.3294705"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2023.3294705","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2023.3294705","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042515126","display_name":"Guinan Li","orcid":"https://orcid.org/0000-0002-2206-0237"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Guinan Li","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106406454","display_name":"Jiajun Deng","orcid":"https://orcid.org/0000-0001-8874-4167"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jiajun Deng","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003895235","display_name":"Mengzhe Geng","orcid":"https://orcid.org/0000-0002-7886-439X"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Mengzhe Geng","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075023049","display_name":"Zengrui Jin","orcid":"https://orcid.org/0000-0002-2637-7880"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zengrui Jin","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106407750","display_name":"Tianzi Wang","orcid":"https://orcid.org/0009-0005-5823-3039"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Tianzi Wang","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026334377","display_name":"Shujie Hu","orcid":"https://orcid.org/0000-0002-8475-4912"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shujie Hu","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065896539","display_name":"Mingyu Cui","orcid":"https://orcid.org/0000-0001-9781-9407"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Mingyu Cui","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019458385","display_name":"Helen Meng","orcid":"https://orcid.org/0000-0002-4427-3532"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Helen Meng","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037109470","display_name":"Xunying Liu","orcid":"https://orcid.org/0000-0001-6725-1160"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xunying Liu","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5042515126"],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":3.338,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.93376623,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"31","issue":null,"first_page":"2707","last_page":"2723"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11233","display_name":"Advanced Adaptive Filtering Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11447","display_name":"Blind Source Separation Techniques","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7941275238990784},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7705713510513306},{"id":"https://openalex.org/keywords/reverberation","display_name":"Reverberation","score":0.5043178796768188},{"id":"https://openalex.org/keywords/front-and-back-ends","display_name":"Front and back ends","score":0.49304935336112976},{"id":"https://openalex.org/keywords/pesq","display_name":"PESQ","score":0.4546405076980591},{"id":"https://openalex.org/keywords/end-to-end-principle","display_name":"End-to-end principle","score":0.4344373941421509},{"id":"https://openalex.org/keywords/speech-enhancement","display_name":"Speech enhancement","score":0.39995378255844116},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.30501341819763184},{"id":"https://openalex.org/keywords/noise-reduction","display_name":"Noise reduction","score":0.23250803351402283},{"id":"https://openalex.org/keywords/acoustics","display_name":"Acoustics","score":0.14570054411888123}],"concepts":[{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7941275238990784},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7705713510513306},{"id":"https://openalex.org/C95851461","wikidata":"https://www.wikidata.org/wiki/Q468809","display_name":"Reverberation","level":2,"score":0.5043178796768188},{"id":"https://openalex.org/C53016008","wikidata":"https://www.wikidata.org/wiki/Q620167","display_name":"Front and back ends","level":2,"score":0.49304935336112976},{"id":"https://openalex.org/C103734657","wikidata":"https://www.wikidata.org/wiki/Q2739975","display_name":"PESQ","level":4,"score":0.4546405076980591},{"id":"https://openalex.org/C74296488","wikidata":"https://www.wikidata.org/wiki/Q2527392","display_name":"End-to-end principle","level":2,"score":0.4344373941421509},{"id":"https://openalex.org/C2776182073","wikidata":"https://www.wikidata.org/wiki/Q7575395","display_name":"Speech enhancement","level":3,"score":0.39995378255844116},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30501341819763184},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.23250803351402283},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.14570054411888123},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2023.3294705","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2023.3294705","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.8100000023841858,"display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G5596890351","display_name":null,"funder_award_id":"ITS/218/21","funder_id":"https://openalex.org/F4320326427","funder_display_name":"Innovation and Technology Fund"}],"funders":[{"id":"https://openalex.org/F4320326427","display_name":"Innovation and Technology Fund","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":103,"referenced_works":["https://openalex.org/W653761051","https://openalex.org/W1503933356","https://openalex.org/W1677182931","https://openalex.org/W1932968309","https://openalex.org/W1935589317","https://openalex.org/W1971168548","https://openalex.org/W2076462394","https://openalex.org/W2125114513","https://openalex.org/W2140571699","https://openalex.org/W2141998673","https://openalex.org/W2148613904","https://openalex.org/W2152536723","https://openalex.org/W2158143227","https://openalex.org/W2159220049","https://openalex.org/W2164502538","https://openalex.org/W2168729028","https://openalex.org/W2194775991","https://openalex.org/W2288645994","https://openalex.org/W2398972335","https://openalex.org/W2517616541","https://openalex.org/W2531409750","https://openalex.org/W2551572271","https://openalex.org/W2589857635","https://openalex.org/W2609317876","https://openalex.org/W2640112133","https://openalex.org/W2696967604","https://openalex.org/W2747732471","https://openalex.org/W2766219058","https://openalex.org/W2774812352","https://openalex.org/W2790326622","https://openalex.org/W2794289187","https://openalex.org/W2803322398","https://openalex.org/W2889224938","https://openalex.org/W2889503488","https://openalex.org/W2890553422","https://openalex.org/W2890952074","https://openalex.org/W2891205112","https://openalex.org/W2900292050","https://openalex.org/W2901702433","https://openalex.org/W2901907199","https://openalex.org/W2905402910","https://openalex.org/W2909607850","https://openalex.org/W2935794029","https://openalex.org/W2938386503","https://openalex.org/W2939360348","https://openalex.org/W2939497224","https://openalex.org/W2939777271","https://openalex.org/W2952218014","https://openalex.org/W2959214850","https://openalex.org/W2963654155","https://openalex.org/W2963785710","https://openalex.org/W2964171275","https://openalex.org/W2964207404","https://openalex.org/W2964283370","https://openalex.org/W2972568703","https://openalex.org/W2972693890","https://openalex.org/W2972728246","https://openalex.org/W2972775954","https://openalex.org/W2973231102","https://openalex.org/W2987026005","https://openalex.org/W3008400075","https://openalex.org/W3008762051","https://openalex.org/W3011424113","https://openalex.org/W3015306655","https://openalex.org/W3015445830","https://openalex.org/W3015612952","https://openalex.org/W3015661721","https://openalex.org/W3015679215","https://openalex.org/W3015785290","https://openalex.org/W3015788098","https://openalex.org/W3016131876","https://openalex.org/W3016176723","https://openalex.org/W3016257794","https://openalex.org/W3024147341","https://openalex.org/W3026111682","https://openalex.org/W3028504434","https://openalex.org/W3032969657","https://openalex.org/W3086154751","https://openalex.org/W3097173744","https://openalex.org/W3097777922","https://openalex.org/W3099177480","https://openalex.org/W3099330747","https://openalex.org/W3136499730","https://openalex.org/W3144062511","https://openalex.org/W3148386121","https://openalex.org/W3160348851","https://openalex.org/W3161697668","https://openalex.org/W3162293946","https://openalex.org/W3163391314","https://openalex.org/W3163793923","https://openalex.org/W3190227394","https://openalex.org/W3196777808","https://openalex.org/W3217643883","https://openalex.org/W4221080854","https://openalex.org/W4224920251","https://openalex.org/W4224934178","https://openalex.org/W4225304491","https://openalex.org/W4225320033","https://openalex.org/W4289665794","https://openalex.org/W4297841468","https://openalex.org/W4297841641","https://openalex.org/W4312356750","https://openalex.org/W6754420807"],"related_works":["https://openalex.org/W2058482658","https://openalex.org/W3016109656","https://openalex.org/W1973895194","https://openalex.org/W2546593254","https://openalex.org/W3135613579","https://openalex.org/W2166831097","https://openalex.org/W4386746628","https://openalex.org/W4388016426","https://openalex.org/W1980687383","https://openalex.org/W3209446892"],"abstract_inverted_index":{"Accurate":[0],"recognition":[1,37,165],"of":[2,23,43,57,152],"cocktail":[3],"party":[4],"speech":[5,33,67,75,87,108,133,145,161,188],"containing":[6],"overlapping":[7],"speakers,":[8],"noise":[9],"and":[10,36,78,89,111,143,164,175,179,197],"reverberation":[11],"remains":[12],"a":[13,40,92],"highly":[14],"challenging":[15],"task":[16],"to":[17,26],"date.":[18],"Motivated":[19],"by":[20,117,173],"the":[21,58,107,123,132,140,153,169],"invariance":[22],"visual":[24,44],"modality":[25],"acoustic":[27],"signal":[28],"corruption,":[29],"an":[30],"audio-visual":[31,159],"multi-channel":[32,160],"separation,":[34,68,162],"dereverberation":[35,76,90,163],"approach":[38],"featuring":[39],"full":[41],"incorporation":[42],"information":[45],"into":[46],"all":[47],"system":[48],"components":[49,114],"is":[50,61,115],"proposed":[51,158],"in":[52,64,91],"this":[53],"paper.":[54],"The":[55,102,157],"efficacy":[56],"video":[59],"input":[60],"consistently":[62,167],"demonstrated":[63],"mask-based":[65,98],"MVDR":[66],"DNN-WPE":[69],"or":[70,94,128,150],"spectral":[71],"mapping":[72],"(SpecM)":[73],"based":[74],"front-end":[77,84,110],"Conformer":[79],"ASR":[80,112,124],"back-end.":[81],"Audio-visual":[82],"integrated":[83],"architectures":[85],"performing":[86],"separation":[88],"pipelined":[93],"joint":[95],"fashion":[96],"via":[97],"WPD":[99],"are":[100],"investigated.":[101],"error":[103,183],"cost":[104,125],"mismatch":[105],"between":[106],"enhancement":[109,134,189],"back-end":[113],"minimized":[116],"end-to-end":[118],"jointly":[119],"fine-tuning":[120],"using":[121,148],"either":[122],"function":[126],"alone,":[127],"its":[129],"interpolation":[130],"with":[131],"loss.":[135],"Experiments":[136],"were":[137,191],"conducted":[138],"on":[139,194],"mixture":[141],"overlapped":[142],"reverberant":[144],"data":[146],"constructed":[147],"simulation":[149],"replay":[151],"Oxford":[154],"LRS2":[155],"dataset.":[156],"systems":[166],"outperformed":[168],"comparable":[170],"audio-only":[171],"baseline":[172],"9.1%":[174],"6.2%":[176],"absolute":[177],"(41.7%":[178],"36.0%":[180],"relative)":[181],"word":[182],"rate":[184],"(WER)":[185],"reductions.":[186],"Consistent":[187],"improvements":[190],"also":[192],"obtained":[193],"PESQ,":[195],"STOI":[196],"SRMR":[198],"scores.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}