{"id":"https://openalex.org/W4291920479","doi":"https://doi.org/10.1109/tmm.2022.3199109","title":"Look&listen: Multi-Modal Correlation Learning for Active Speaker Detection and Speech Enhancement","display_name":"Look&listen: Multi-Modal Correlation Learning for Active Speaker Detection and Speech Enhancement","publication_year":2022,"publication_date":"2022-08-15","ids":{"openalex":"https://openalex.org/W4291920479","doi":"https://doi.org/10.1109/tmm.2022.3199109"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2022.3199109","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2022.3199109","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006922165","display_name":"Junwen Xiong","orcid":"https://orcid.org/0000-0001-5180-1295"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junwen Xiong","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-5180-1295","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043389001","display_name":"Zhou Yu","orcid":"https://orcid.org/0000-0002-6356-8360"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Zhou","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-6356-8360","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100364068","display_name":"Peng Zhang","orcid":"https://orcid.org/0000-0001-9690-7026"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Zhang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi'an, China","Ningbo Institute of Northwestern Polytechnical University, Ningbo, China"],"raw_orcid":"https://orcid.org/0000-0001-9690-7026","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]},{"raw_affiliation_string":"Ningbo Institute of Northwestern Polytechnical University, Ningbo, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100668966","display_name":"Lei Xie","orcid":"https://orcid.org/0000-0001-8234-0823"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Xie","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-8234-0823","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004584268","display_name":"Wei Huang","orcid":"https://orcid.org/0000-0002-0541-8612"},"institutions":[{"id":"https://openalex.org/I141649914","display_name":"Nanchang University","ror":"https://ror.org/042v6xz23","country_code":"CN","type":"education","lineage":["https://openalex.org/I141649914"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Huang","raw_affiliation_strings":["School of Mathematics and Computer Sciences, Nanchang University, Nanchang, China"],"raw_orcid":"https://orcid.org/0000-0002-0541-8612","affiliations":[{"raw_affiliation_string":"School of Mathematics and Computer Sciences, Nanchang University, Nanchang, China","institution_ids":["https://openalex.org/I141649914"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014604035","display_name":"Yufei Zha","orcid":"https://orcid.org/0000-0001-5013-2501"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yufei Zha","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi'an, China","Ningbo Institute of Northwestern Polytechnical University, Ningbo, China"],"raw_orcid":"https://orcid.org/0000-0001-5013-2501","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]},{"raw_affiliation_string":"Ningbo Institute of Northwestern Polytechnical University, Ningbo, China","institution_ids":["https://openalex.org/I17145004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.2522,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.92813106,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"25","issue":null,"first_page":"5800","last_page":"5812"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11233","display_name":"Advanced Adaptive Filtering Techniques","score":0.9819999933242798,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.839775025844574},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6064426898956299},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"Multi-task learning","score":0.552692711353302},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.5173558592796326},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5151761770248413},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4616275131702423},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.38546207547187805}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.839775025844574},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6064426898956299},{"id":"https://openalex.org/C28006648","wikidata":"https://www.wikidata.org/wiki/Q6934509","display_name":"Multi-task learning","level":3,"score":0.552692711353302},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.5173558592796326},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5151761770248413},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4616275131702423},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.38546207547187805},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2022.3199109","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2022.3199109","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G210655010","display_name":null,"funder_award_id":"2021J048","funder_id":"https://openalex.org/F4320332587","funder_display_name":"Natural Science Foundation of Ningbo"},{"id":"https://openalex.org/G2971322799","display_name":"\u57fa\u4e8e\u5168\u5c3a\u5ea6\u7f51\u7edc\u6df1\u5ea6\u5ea6\u91cf\u5b66\u4e60\u4e0e\u56fe\u50cf\u6df1\u5ea6\u63a8\u884d\u7684\u5931\u667a\u75c7\u8f85\u52a9\u8bca\u65ad\u7814\u7a76","funder_award_id":"61862043","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3444584140","display_name":"\u57fa\u4e8e\u591a\u5f62\u6001\u6311\u6218\u6837\u672c\u7ea7\u8054\u751f\u6210\u5b66\u4e60\u7684\u590d\u5408\u8ddf\u8e2a\u6846\u67b6\u6a21\u578b\u7814\u7a76","funder_award_id":"61971352","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G44360340","display_name":null,"funder_award_id":"2021J049","funder_id":"https://openalex.org/F4320332587","funder_display_name":"Natural Science Foundation of Ningbo"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320332587","display_name":"Natural Science Foundation of Ningbo","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1522734439","https://openalex.org/W1552314771","https://openalex.org/W1578985305","https://openalex.org/W1592234069","https://openalex.org/W2012994257","https://openalex.org/W2023100145","https://openalex.org/W2037960784","https://openalex.org/W2046394129","https://openalex.org/W2055447045","https://openalex.org/W2058580374","https://openalex.org/W2076029968","https://openalex.org/W2081144555","https://openalex.org/W2090088747","https://openalex.org/W2109788549","https://openalex.org/W2157187516","https://openalex.org/W2187089797","https://openalex.org/W2194775991","https://openalex.org/W2219249508","https://openalex.org/W2221409856","https://openalex.org/W2259890326","https://openalex.org/W2323437298","https://openalex.org/W2330149154","https://openalex.org/W2519091744","https://openalex.org/W2547411571","https://openalex.org/W2604379605","https://openalex.org/W2610846088","https://openalex.org/W2749694333","https://openalex.org/W2752782242","https://openalex.org/W2808631503","https://openalex.org/W2832876791","https://openalex.org/W2890952074","https://openalex.org/W2891205112","https://openalex.org/W2900292050","https://openalex.org/W2952218014","https://openalex.org/W2962960500","https://openalex.org/W2963082324","https://openalex.org/W2964171275","https://openalex.org/W2965002127","https://openalex.org/W2971680695","https://openalex.org/W3007589762","https://openalex.org/W3008402854","https://openalex.org/W3010594275","https://openalex.org/W3011519842","https://openalex.org/W3016098309","https://openalex.org/W3034658206","https://openalex.org/W3034702511","https://openalex.org/W3094378471","https://openalex.org/W3097777922","https://openalex.org/W3099202502","https://openalex.org/W3099330747","https://openalex.org/W3116298410","https://openalex.org/W3118593445","https://openalex.org/W3119269912","https://openalex.org/W3136499730","https://openalex.org/W3160005627","https://openalex.org/W3160672713","https://openalex.org/W3172472082","https://openalex.org/W3181255780","https://openalex.org/W3182657421","https://openalex.org/W3186090335","https://openalex.org/W3189964604","https://openalex.org/W3207207922","https://openalex.org/W3209552243","https://openalex.org/W3214910576","https://openalex.org/W4200436929","https://openalex.org/W4200631136","https://openalex.org/W4214601882","https://openalex.org/W4214701094","https://openalex.org/W4226105092","https://openalex.org/W4289665794","https://openalex.org/W6631190155","https://openalex.org/W6676413359","https://openalex.org/W6688816777","https://openalex.org/W6754420807","https://openalex.org/W6774509113","https://openalex.org/W6780226713","https://openalex.org/W6785273600","https://openalex.org/W6798523648","https://openalex.org/W7028336066"],"related_works":["https://openalex.org/W2237537322","https://openalex.org/W2950678851","https://openalex.org/W4301248618","https://openalex.org/W2165343651","https://openalex.org/W2242427765","https://openalex.org/W2075830955","https://openalex.org/W2343790552","https://openalex.org/W2951720331","https://openalex.org/W3111398917","https://openalex.org/W4385507578"],"abstract_inverted_index":{"Active":[0],"speaker":[1,110,245],"detection":[2,111,246],"and":[3,49,76,112,148,157,215,247],"speech":[4,113,249],"enhancement":[5,114,250],"have":[6,68],"become":[7],"two":[8],"increasingly":[9],"attractive":[10],"topics":[11],"in":[12,31,52,99,121,132,145,202,260],"audio-visual":[13,87,100,119,128,161,200,248],"scenario":[14],"understanding.":[15],"According":[16],"to":[17,33,40,94,107,141,154,179,197,232],"their":[18],"respective":[19],"characteristics,":[20],"the":[21,41,45,53,59,84,96,125,136,159,181,192,210,216,225,236],"scheme":[22],"of":[23,55,58,86,118,127,130,184],"independently":[24],"designed":[25],"architecture":[26],"has":[27],"been":[28],"widely":[29],"used":[30,153],"correspondence":[32],"each":[34],"single":[35],"task.":[36],"This":[37],"may":[38],"lead":[39],"representation":[42],"learned":[43,198,223],"by":[44,224],"model":[46,158,194,227],"being":[47],"task-specific,":[48],"inevitably":[50],"result":[51],"lack":[54],"generalization":[56,259],"ability":[57],"feature":[60],"based":[61],"on":[62,188,251],"multi-modal":[63,97,174,185],"modeling.":[64],"More":[65],"recent":[66],"studies":[67],"shown":[69],"that":[70,209],"establishing":[71],"cross-modal":[72,168,189],"relationship":[73],"between":[74,212],"auditory":[75],"visual":[77,149],"stream":[78],"is":[79,105,139,152,177,195],"a":[80,92,102,167,172,203,240,257],"promising":[81],"solution":[82],"for":[83,243],"challenge":[85],"multi-task":[88],"learning.":[89],"Therefore,":[90],"as":[91],"motivation":[93],"bridge":[95],"associations":[98,217],"tasks,":[101],"unified":[103],"framework":[104],"proposed":[106,137,193,237],"achieve":[108],"target":[109],"with":[115,256],"joint":[116],"learning":[117],"modeling":[120],"this":[122],"study.":[123],"With":[124],"assistance":[126],"channels":[129],"videos":[131],"challenging":[133],"real-world":[134],"scenarios,":[135],"method":[138],"able":[140],"exploit":[142],"inherent":[143],"correlations":[144,211],"both":[146],"audio":[147],"signals,":[150],"which":[151],"further":[155],"anticipate":[156],"temporal":[160],"relationships":[162],"across":[163],"spatial-temporal":[164],"space":[165],"via":[166],"conformer.":[169],"In":[170,230],"addition,":[171],"plug-and-play":[173],"layer":[175],"normalization":[176],"introduced":[178],"alleviate":[180],"distribution":[182],"misalignment":[183],"features.":[186],"Based":[187],"circulant":[190],"fusion,":[191],"capable":[196],"all":[199],"representations":[201],"holistic":[204],"process.":[205],"Substantial":[206],"experiments":[207],"demonstrate":[208],"different":[213],"modalities":[214],"among":[218],"diverse":[219,261],"tasks":[220],"can":[221],"be":[222],"optimized":[226],"more":[228],"effectively.":[229],"comparison":[231],"other":[233],"state-of-the-art":[234],"works,":[235],"work":[238],"shows":[239],"superior":[241],"performance":[242],"active":[244],"three":[252],"benchmark":[253],"datasets,":[254],"also":[255],"favorable":[258],"challenges.":[262]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}