{"id":"https://openalex.org/W2973231102","doi":"https://doi.org/10.1109/jstsp.2020.2987209","title":"Audio-Visual Speech Separation and Dereverberation With a Two-Stage Multimodal Network","display_name":"Audio-Visual Speech Separation and Dereverberation With a Two-Stage Multimodal Network","publication_year":2020,"publication_date":"2020-03-01","ids":{"openalex":"https://openalex.org/W2973231102","doi":"https://doi.org/10.1109/jstsp.2020.2987209","mag":"2973231102"},"language":"en","primary_location":{"id":"doi:10.1109/jstsp.2020.2987209","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2020.2987209","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1909.07352","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058209019","display_name":"Ke Tan","orcid":"https://orcid.org/0000-0001-5073-8060"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ke Tan","raw_affiliation_strings":["Department of Computer Science and Engineering, The Ohio State University, Columbus, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, The Ohio State University, Columbus, USA","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101467338","display_name":"Yong Xu","orcid":"https://orcid.org/0000-0003-4944-6890"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yong Xu","raw_affiliation_strings":["Tencent AI Lab, Bellevue, USA"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Bellevue, USA","institution_ids":["https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101785327","display_name":"Shi-Xiong Zhang","orcid":"https://orcid.org/0009-0007-7803-2407"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shi-Xiong Zhang","raw_affiliation_strings":["Tencent AI Lab, Bellevue, USA"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Bellevue, USA","institution_ids":["https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106407019","display_name":"Meng Yu","orcid":"https://orcid.org/0000-0002-0031-9156"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Meng Yu","raw_affiliation_strings":["Tencent AI Lab, Bellevue, USA"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Bellevue, USA","institution_ids":["https://openalex.org/I4210108985"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034476404","display_name":"Dong Yu","orcid":"https://orcid.org/0000-0003-0520-6844"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dong Yu","raw_affiliation_strings":["Tencent AI Lab, Bellevue, USA"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Bellevue, USA","institution_ids":["https://openalex.org/I4210108985"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5058209019"],"corresponding_institution_ids":["https://openalex.org/I52357470"],"apc_list":null,"apc_paid":null,"fwci":7.183,"has_fulltext":false,"cited_by_count":65,"citation_normalized_percentile":{"value":0.97958252,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"14","issue":"3","first_page":"542","last_page":"553"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10283","display_name":"Hearing Loss and Rehabilitation","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11233","display_name":"Advanced Adaptive Filtering Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8340745568275452},{"id":"https://openalex.org/keywords/pesq","display_name":"PESQ","score":0.7707011699676514},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7404041290283203},{"id":"https://openalex.org/keywords/reverberation","display_name":"Reverberation","score":0.6886612772941589},{"id":"https://openalex.org/keywords/speech-enhancement","display_name":"Speech enhancement","score":0.687660276889801},{"id":"https://openalex.org/keywords/active-listening","display_name":"Active listening","score":0.6049551367759705},{"id":"https://openalex.org/keywords/intelligibility","display_name":"Intelligibility (philosophy)","score":0.5871531963348389},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.47889748215675354},{"id":"https://openalex.org/keywords/background-noise","display_name":"Background noise","score":0.477030485868454},{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.44922393560409546},{"id":"https://openalex.org/keywords/voice-activity-detection","display_name":"Voice activity detection","score":0.438412070274353},{"id":"https://openalex.org/keywords/source-separation","display_name":"Source separation","score":0.42008236050605774},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.4142783284187317},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.372604638338089},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.30023282766342163},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.13211697340011597},{"id":"https://openalex.org/keywords/acoustics","display_name":"Acoustics","score":0.11457571387290955},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.06471401453018188}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8340745568275452},{"id":"https://openalex.org/C103734657","wikidata":"https://www.wikidata.org/wiki/Q2739975","display_name":"PESQ","level":4,"score":0.7707011699676514},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7404041290283203},{"id":"https://openalex.org/C95851461","wikidata":"https://www.wikidata.org/wiki/Q468809","display_name":"Reverberation","level":2,"score":0.6886612772941589},{"id":"https://openalex.org/C2776182073","wikidata":"https://www.wikidata.org/wiki/Q7575395","display_name":"Speech enhancement","level":3,"score":0.687660276889801},{"id":"https://openalex.org/C177291462","wikidata":"https://www.wikidata.org/wiki/Q423038","display_name":"Active listening","level":2,"score":0.6049551367759705},{"id":"https://openalex.org/C60048801","wikidata":"https://www.wikidata.org/wiki/Q1433889","display_name":"Intelligibility (philosophy)","level":2,"score":0.5871531963348389},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.47889748215675354},{"id":"https://openalex.org/C100675267","wikidata":"https://www.wikidata.org/wiki/Q1371624","display_name":"Background noise","level":2,"score":0.477030485868454},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.44922393560409546},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.438412070274353},{"id":"https://openalex.org/C2776864781","wikidata":"https://www.wikidata.org/wiki/Q52617913","display_name":"Source separation","level":2,"score":0.42008236050605774},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4142783284187317},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.372604638338089},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30023282766342163},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.13211697340011597},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.11457571387290955},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.06471401453018188},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C170154142","wikidata":"https://www.wikidata.org/wiki/Q150737","display_name":"Architectural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/jstsp.2020.2987209","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jstsp.2020.2987209","pdf_url":null,"source":{"id":"https://openalex.org/S42167783","display_name":"IEEE Journal of Selected Topics in Signal Processing","issn_l":"1932-4553","issn":["1932-4553","1941-0484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Selected Topics in Signal Processing","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1909.07352","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1909.07352","pdf_url":"https://arxiv.org/pdf/1909.07352","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1909.07352","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1909.07352","pdf_url":"https://arxiv.org/pdf/1909.07352","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.5199999809265137,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":72,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1552314771","https://openalex.org/W1558945660","https://openalex.org/W1677182931","https://openalex.org/W1836465849","https://openalex.org/W1983812858","https://openalex.org/W1986572644","https://openalex.org/W1991139021","https://openalex.org/W2002621203","https://openalex.org/W2022870009","https://openalex.org/W2070126272","https://openalex.org/W2099731107","https://openalex.org/W2117678320","https://openalex.org/W2135956791","https://openalex.org/W2141998673","https://openalex.org/W2144763279","https://openalex.org/W2144786298","https://openalex.org/W2164502538","https://openalex.org/W2167734090","https://openalex.org/W2168379380","https://openalex.org/W2194775991","https://openalex.org/W2221409856","https://openalex.org/W2233657726","https://openalex.org/W2421316443","https://openalex.org/W2460742184","https://openalex.org/W2516001803","https://openalex.org/W2519091744","https://openalex.org/W2542605056","https://openalex.org/W2561557072","https://openalex.org/W2577762507","https://openalex.org/W2734774145","https://openalex.org/W2735663686","https://openalex.org/W2788241093","https://openalex.org/W2796992393","https://openalex.org/W2804093409","https://openalex.org/W2889384217","https://openalex.org/W2889540509","https://openalex.org/W2890820256","https://openalex.org/W2891355459","https://openalex.org/W2897371647","https://openalex.org/W2900292050","https://openalex.org/W2909607850","https://openalex.org/W2911394281","https://openalex.org/W2952218014","https://openalex.org/W2962715207","https://openalex.org/W2962866211","https://openalex.org/W2962960500","https://openalex.org/W2963082324","https://openalex.org/W2963103134","https://openalex.org/W2963317762","https://openalex.org/W2964121744","https://openalex.org/W2964171275","https://openalex.org/W2964207404","https://openalex.org/W2964238697","https://openalex.org/W2966212213","https://openalex.org/W2972436155","https://openalex.org/W2973062255","https://openalex.org/W2980451698","https://openalex.org/W3008400075","https://openalex.org/W3099330747","https://openalex.org/W3123318516","https://openalex.org/W3124972797","https://openalex.org/W4289665794","https://openalex.org/W4296927107","https://openalex.org/W4301143517","https://openalex.org/W4307196995","https://openalex.org/W4394666973","https://openalex.org/W6631190155","https://openalex.org/W6638667902","https://openalex.org/W6717465090","https://openalex.org/W6766475400","https://openalex.org/W6780226713"],"related_works":["https://openalex.org/W2058482658","https://openalex.org/W3016109656","https://openalex.org/W3135613579","https://openalex.org/W1973895194","https://openalex.org/W4388016426","https://openalex.org/W1980687383","https://openalex.org/W4386746628","https://openalex.org/W2546593254","https://openalex.org/W2166831097","https://openalex.org/W2373767407"],"abstract_inverted_index":{"Background":[0],"noise,":[1,33],"interfering":[2,34,79],"speech":[3,10,21,30,35,80],"and":[4,23,36,57,78,85,104,133,139,153],"room":[5,37,91],"reverberation":[6,92],"frequently":[7],"distort":[8],"target":[9,29],"in":[11,81,93,151,157],"real":[12],"listening":[13],"environments.":[14],"In":[15,39],"this":[16,43],"study,":[17],"we":[18,47],"address":[19],"joint":[20,108],"separation":[22,70],"dereverberation,":[24],"which":[25,110],"aims":[26],"to":[27,41,74,89],"separate":[28],"from":[31],"background":[32,76],"reverberation.":[38],"order":[40],"tackle":[42],"fundamentally":[44],"difficult":[45],"problem,":[46],"propose":[48],"a":[49,65,69,86,114,148,154],"novel":[50],"multimodal":[51,126],"network":[52,62,127,146,165],"that":[53,123,144],"exploits":[54],"both":[55],"audio":[56],"visual":[58],"signals.":[59],"The":[60,97],"proposed":[61,125],"architecture":[63,166],"adopts":[64],"two-stage":[66,140],"strategy,":[67],"where":[68],"module":[71,88],"is":[72,111],"employed":[73],"attenuate":[75],"noise":[77],"the":[82,94,124,160,170,173],"first":[83,101],"stage":[84],"dereverberation":[87],"suppress":[90],"second":[95],"stage.":[96],"two":[98],"modules":[99],"are":[100],"trained":[102],"separately,":[103],"then":[105],"integrated":[106],"for":[107],"training,":[109],"based":[112],"on":[113],"new":[115],"multi-objective":[116],"loss":[117],"function.":[118],"Our":[119],"experimental":[120],"results":[121],"show":[122],"yields":[128],"consistently":[129],"better":[130],"objective":[131],"intelligibility":[132],"perceptual":[134],"quality":[135],"than":[136],"several":[137],"one-stage":[138],"baselines.":[141],"We":[142],"find":[143],"our":[145,164],"achieves":[147],"21.10%":[149],"improvement":[150,156],"ESTOI":[152],"0.79":[155],"PESQ":[158],"over":[159],"unprocessed":[161],"mixtures.":[162],"Moreover,":[163],"does":[167],"not":[168],"require":[169],"knowledge":[171],"of":[172,175],"number":[174],"speakers.":[176]},"counts_by_year":[{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":16},{"year":2020,"cited_by_count":7}],"updated_date":"2026-04-03T22:45:19.894376","created_date":"2025-10-10T00:00:00"}