{"id":"https://openalex.org/W3160348851","doi":"https://doi.org/10.1109/taslp.2021.3078883","title":"Audio-Visual Multi-Channel Integration and Recognition of Overlapped Speech","display_name":"Audio-Visual Multi-Channel Integration and Recognition of Overlapped Speech","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3160348851","doi":"https://doi.org/10.1109/taslp.2021.3078883","mag":"3160348851"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2021.3078883","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2021.3078883","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004643540","display_name":"Jianwei Yu","orcid":"https://orcid.org/0000-0002-2449-1436"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Jianwei Yu","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong","Tencent AI Lab, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-2449-1436","affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Tencent AI Lab, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101785327","display_name":"Shi-Xiong Zhang","orcid":"https://orcid.org/0009-0007-7803-2407"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shi-Xiong Zhang","raw_affiliation_strings":["Tencent AI Lab, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100672630","display_name":"Bo Wu","orcid":"https://orcid.org/0000-0002-3126-5103"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bo Wu","raw_affiliation_strings":["Tencent AI Lab, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066070154","display_name":"Shansong Liu","orcid":"https://orcid.org/0000-0001-6202-5615"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shansong Liu","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0001-6202-5615","affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045355404","display_name":"Shoukang Hu","orcid":"https://orcid.org/0000-0002-3345-6923"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shoukang Hu","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-3345-6923","affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003895235","display_name":"Mengzhe Geng","orcid":"https://orcid.org/0000-0002-7886-439X"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Mengzhe Geng","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037109470","display_name":"Xunying Liu","orcid":"https://orcid.org/0000-0001-6725-1160"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xunying Liu","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0001-6725-1160","affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019458385","display_name":"Helen Meng","orcid":"https://orcid.org/0000-0002-4427-3532"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Helen Meng","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034476404","display_name":"Dong Yu","orcid":"https://orcid.org/0000-0003-0520-6844"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dong Yu","raw_affiliation_strings":["Tencent AI Lab, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-0520-6844","affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210108985"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.9279,"has_fulltext":false,"cited_by_count":23,"citation_normalized_percentile":{"value":0.91521007,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"29","issue":null,"first_page":"2067","last_page":"2082"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11233","display_name":"Advanced Adaptive Filtering Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7767125964164734},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7713749408721924},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.49738410115242004},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41233766078948975},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.37235027551651}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7767125964164734},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7713749408721924},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.49738410115242004},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41233766078948975},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.37235027551651},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2021.3078883","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2021.3078883","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.8199999928474426}],"awards":[{"id":"https://openalex.org/G3571483920","display_name":null,"funder_award_id":"MMT-p1-19","funder_id":"https://openalex.org/F4320335138","funder_display_name":"Shun Hing Institute of Advanced Engineering"}],"funders":[{"id":"https://openalex.org/F4320335138","display_name":"Shun Hing Institute of Advanced Engineering","ror":"https://ror.org/00t33hh48"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W1489488446","https://openalex.org/W1600744878","https://openalex.org/W1677182931","https://openalex.org/W2022799064","https://openalex.org/W2060108923","https://openalex.org/W2067584370","https://openalex.org/W2076462394","https://openalex.org/W2102764819","https://openalex.org/W2121486117","https://openalex.org/W2127141656","https://openalex.org/W2135823751","https://openalex.org/W2148613904","https://openalex.org/W2158143227","https://openalex.org/W2168729028","https://openalex.org/W2194775991","https://openalex.org/W2288645994","https://openalex.org/W2289394825","https://openalex.org/W2289912446","https://openalex.org/W2291877678","https://openalex.org/W2398042854","https://openalex.org/W2398972335","https://openalex.org/W2508092598","https://openalex.org/W2514741789","https://openalex.org/W2517616541","https://openalex.org/W2531409750","https://openalex.org/W2551572271","https://openalex.org/W2589857635","https://openalex.org/W2640112133","https://openalex.org/W2734774145","https://openalex.org/W2803322398","https://openalex.org/W2808939837","https://openalex.org/W2889282842","https://openalex.org/W2889503488","https://openalex.org/W2890952074","https://openalex.org/W2891984996","https://openalex.org/W2909607850","https://openalex.org/W2935794029","https://openalex.org/W2939360348","https://openalex.org/W2939497224","https://openalex.org/W2944972166","https://openalex.org/W2952218014","https://openalex.org/W2952746495","https://openalex.org/W2962866211","https://openalex.org/W2963420272","https://openalex.org/W2963785710","https://openalex.org/W2964171275","https://openalex.org/W2964283370","https://openalex.org/W2972513594","https://openalex.org/W2972693890","https://openalex.org/W2973071728","https://openalex.org/W2973231102","https://openalex.org/W2995166068","https://openalex.org/W3004309045","https://openalex.org/W3006974783","https://openalex.org/W3008400075","https://openalex.org/W3008762051","https://openalex.org/W3011424113","https://openalex.org/W3015372568","https://openalex.org/W3015383493","https://openalex.org/W3015834770","https://openalex.org/W3096408984","https://openalex.org/W3097798370","https://openalex.org/W3097966334","https://openalex.org/W3099330747","https://openalex.org/W4285719527","https://openalex.org/W4289665794","https://openalex.org/W6762182681"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2033914206","https://openalex.org/W2042327336"],"abstract_inverted_index":{"Automatic":[0],"speech":[1,18,87,105,201],"recognition":[2,15,74,91,134,177,224],"(ASR)":[3],"technologies":[4],"have":[5],"been":[6],"significantly":[7],"advanced":[8],"in":[9,36],"the":[10,42,52,59,63,125,130,133,136,148,157,173,180,209,220,232],"past":[11],"few":[12],"decades.":[13],"However,":[14],"of":[16,44,94,102,147,208],"overlapped":[17,77,200],"remains":[19],"a":[20,82,86,143],"highly":[21],"challenging":[22],"task":[23],"to":[24,47,57,151,188,238],"date.":[25],"To":[26,123],"this":[27,67],"end,":[28],"multi-channel":[29,72,104,176,183,223],"microphone":[30],"array":[31],"data":[32],"are":[33,121,216],"widely":[34],"used":[35],"current":[37],"ASR":[38,184],"systems.":[39],"Motivated":[40],"by":[41,186],"invariance":[43],"visual":[45],"modality":[46],"acoustic":[48],"signal":[49,150],"corruption":[50],"and":[51,90,114,132,192],"additional":[53,97],"cues":[54],"they":[55],"provide":[56],"separate":[58],"target":[60],"speaker":[61],"from":[62,81],"interfering":[64],"sound":[65],"sources,":[66],"paper":[68],"presents":[69],"an":[70],"audio-visual":[71,103,175,222],"based":[73,109],"system":[75,138,178,185,225],"for":[76],"speech.":[78],"It":[79],"benefits":[80],"tight":[83],"integration":[84,119],"between":[85,129],"separation":[88,106,131],"front-end":[89,107],"back-end,":[92],"both":[93],"which":[95],"incorporate":[96],"video":[98,229],"input.":[99],"A":[100],"series":[101],"components":[108],"on":[110,199],"TF":[111],"masking,":[112],"Filter&Sum":[113],"mask-based":[115],"MVDR":[116],"neural":[117],"channel":[118],"approaches":[120],"developed.":[122],"reduce":[124],"error":[126],"cost":[127],"mismatch":[128],"components,":[135],"entire":[137],"is":[139],"jointly":[140],"fine-tuned":[141],"using":[142,203,219,227],"multi-task":[144],"criterion":[145],"interpolation":[146],"scale-invariant":[149],"noise":[152],"ratio":[153],"(Si-SNR)":[154],"with":[155,231],"either":[156,204],"connectionist":[158],"temporal":[159],"classification":[160],"(CTC),":[161],"or":[162,206],"lattice-free":[163],"maximum":[164],"mutual":[165],"information":[166],"(LF-MMI)":[167],"loss":[168],"function.":[169],"Experiments":[170],"suggest":[171],"that:":[172],"proposed":[174,221],"outperforms":[179],"baseline":[181],"audio-only":[182],"up":[187,237],"8.04%":[189],"(31.68%":[190],"relative)":[191,195],"22.86%":[193],"(58.51%":[194],"absolute":[196],"WER":[197],"reduction":[198],"constructed":[202],"simulation":[205],"replaying":[207],"LRS2":[210],"dataset":[211],"respectively.":[212],"Consistent":[213],"performance":[214],"improvements":[215],"also":[217],"obtained":[218],"when":[226],"occluded":[228],"input":[230],"lip":[233],"region":[234],"randomly":[235],"covered":[236],"60%.":[239]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":4}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}