{"id":"https://openalex.org/W4393152829","doi":"https://doi.org/10.1609/aaai.v38i17.29951","title":"Multichannel AV-wav2vec2: A Framework for Learning Multichannel Multi-Modal Speech Representation","display_name":"Multichannel AV-wav2vec2: A Framework for Learning Multichannel Multi-Modal Speech Representation","publication_year":2024,"publication_date":"2024-03-24","ids":{"openalex":"https://openalex.org/W4393152829","doi":"https://doi.org/10.1609/aaai.v38i17.29951"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v38i17.29951","doi":"https://doi.org/10.1609/aaai.v38i17.29951","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i17.29951","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29951/31662","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29951/31662","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045966396","display_name":"Qiushi Zhu","orcid":"https://orcid.org/0000-0002-1196-7781","observed_orcids":["https://orcid.org/0000-0002-1196-7781"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiushi Zhu","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100436848","display_name":"Jie Zhang","orcid":"https://orcid.org/0000-0003-1124-0854","observed_orcids":["https://orcid.org/0000-0003-1124-0854"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Zhang","raw_affiliation_strings":["University of Science and Technology of China (USTC)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China (USTC)","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yu Gu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Gu","raw_affiliation_strings":["Tencent AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008147837","display_name":"Yuchen Hu","orcid":"https://orcid.org/0000-0002-1244-4683","observed_orcids":["https://orcid.org/0000-0002-1244-4683"]},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yuchen Hu","raw_affiliation_strings":["Nanyang Technological University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110935514","display_name":"Lirong Dai","orcid":"https://orcid.org/0000-0002-0859-2827","observed_orcids":["https://orcid.org/0000-0002-0859-2827"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lirong Dai","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.2904,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.91005407,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":"38","issue":"17","first_page":"19768","last_page":"19776"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.5845000147819519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.5845000147819519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.4124000072479248,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/audio-visual-speech-recognition","display_name":"audio-visual speech recognition","score":0.8429999947547913},{"id":"https://openalex.org/keywords/visual-speech-recognition","display_name":"visual speech recognition","score":0.9010000228881836},{"id":"https://openalex.org/keywords/far-field-speech","display_name":"far-field speech","score":0.7179999947547913},{"id":"https://openalex.org/keywords/contrastive-learning","display_name":"contrastive learning","score":0.8939999938011169},{"id":"https://openalex.org/keywords/chinese-speech","display_name":"Chinese speech","score":0.3709999918937683}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6720021963119507},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5716509819030762},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5459562540054321},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5247823596000671},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3569793403148651},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.05891576409339905},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v38i17.29951","doi":"https://doi.org/10.1609/aaai.v38i17.29951","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i17.29951","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29951/31662","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2401.03468","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.03468","pdf_url":"https://arxiv.org/pdf/2401.03468","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/29951","doi":null,"is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/29951","pdf_url":null,"source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.2401.03468","doi":"https://doi.org/10.48550/arxiv.2401.03468","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2401.03468","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v38i17.29951","doi":"https://doi.org/10.1609/aaai.v38i17.29951","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i17.29951","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29951/31662","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G634576251","display_name":null,"funder_award_id":"62101523","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320325599","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393152829.pdf","grobid_xml":"https://content.openalex.org/works/W4393152829.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W1982728343","https://openalex.org/W1992475611","https://openalex.org/W2064675550","https://openalex.org/W2109886035","https://openalex.org/W2115252128","https://openalex.org/W2148613904","https://openalex.org/W2398042854","https://openalex.org/W2398972335","https://openalex.org/W2506203739","https://openalex.org/W2796868621","https://openalex.org/W2902894493","https://openalex.org/W2921214093","https://openalex.org/W2921496354","https://openalex.org/W2963197115","https://openalex.org/W3003133812","https://openalex.org/W3004928994","https://openalex.org/W3036601975","https://openalex.org/W3097777922","https://openalex.org/W3160207687","https://openalex.org/W3162293946","https://openalex.org/W3169320628","https://openalex.org/W3169722760","https://openalex.org/W3209984917","https://openalex.org/W4221153068","https://openalex.org/W4224933780","https://openalex.org/W4291566970","https://openalex.org/W4297841387","https://openalex.org/W4297841641","https://openalex.org/W4297841834","https://openalex.org/W4307286264","https://openalex.org/W4372317224","https://openalex.org/W4376481237","https://openalex.org/W4385571549","https://openalex.org/W4385571863","https://openalex.org/W4385573012","https://openalex.org/W4385767994","https://openalex.org/W4391021811","https://openalex.org/W6601026004","https://openalex.org/W6605278787"],"related_works":["https://openalex.org/W4391021811","https://openalex.org/W4372259972","https://openalex.org/W4403622721","https://openalex.org/W7163644853","https://openalex.org/W4311553423","https://openalex.org/W3040260790","https://openalex.org/W4221153521","https://openalex.org/W3157916917","https://openalex.org/W3212414183","https://openalex.org/W4417539927"],"abstract_inverted_index":{"Self-supervised":[0],"speech":[1,20,25,50,64,78,125,171,175,179],"pre-training":[2],"methods":[3],"have":[4],"developed":[5],"rapidly":[6],"in":[7,67,70,107,123,158],"recent":[8],"years,":[9],"which":[10,83,138],"show":[11],"to":[12,62,98,117,142,161],"be":[13],"very":[14],"effective":[15],"for":[16,45],"many":[17],"near-field":[18],"single-channel":[19,135],"tasks.":[21,187],"However,":[22],"far-field":[23,46],"multichannel":[24,34,47,76,87,147,155],"processing":[26,51],"is":[27,139],"suffering":[28],"from":[29],"the":[30,75,120,144,163,166],"scarcity":[31],"of":[32,42,146,165],"labeled":[33],"data":[35,89],"and":[36,48,86,103,111,182],"complex":[37],"ambient":[38],"noises.":[39],"The":[40],"efficacy":[41],"self-supervised":[43,79],"learning":[44,80],"multi-modal":[49,77,148,156],"has":[52],"not":[53],"been":[54],"well":[55],"explored.":[56],"Considering":[57],"that":[58],"visual":[59,105,178],"information":[60,122],"helps":[61],"improve":[63,143],"recognition":[65,172,176,180],"performance":[66,145],"noisy":[68],"scenes,":[69],"this":[71],"work":[72],"we":[73,93,132,151],"propose":[74,94],"framework":[81],"AV-wav2vec2,":[82],"utilizes":[84],"video":[85],"audio":[88,101,136],"as":[90,114],"inputs.":[91],"First,":[92],"a":[95,104,153],"multi-path":[96],"structure":[97],"process":[99],"multi-channel":[100,124],"streams":[102],"stream":[106],"parallel,":[108],"with":[109],"intra-,":[110],"inter-channel":[112],"contrastive":[113,130],"training":[115],"targets":[116],"fully":[118],"exploit":[119],"rich":[121],"data.":[126],"Second,":[127],"based":[128],"on":[129,169],"learning,":[131],"use":[133,152],"additional":[134],"data,":[137],"trained":[140],"jointly":[141],"representation.":[149],"Finally,":[150],"Chinese":[154],"dataset":[157],"real":[159],"scenarios":[160],"validate":[162],"effectiveness":[164],"proposed":[167],"method":[168],"audio-visual":[170,183],"(AVSR),":[173],"automatic":[174],"(ASR),":[177],"(VSR)":[181],"speaker":[184],"diarization":[185],"(AVSD)":[186]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-03-26T00:00:00"}