{"id":"https://openalex.org/W4392901809","doi":"https://doi.org/10.1109/taslp.2024.3375641","title":"Unified Cross-Modal Attention: Robust Audio-Visual Speech Recognition and Beyond","display_name":"Unified Cross-Modal Attention: Robust Audio-Visual Speech Recognition and Beyond","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4392901809","doi":"https://doi.org/10.1109/taslp.2024.3375641"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2024.3375641","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3375641","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100658801","display_name":"Jiahong Li","orcid":"https://orcid.org/0009-0002-0639-1519"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiahong Li","raw_affiliation_strings":["Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0002-0639-1519","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090224430","display_name":"Chenda Li","orcid":"https://orcid.org/0000-0003-0299-9914"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenda Li","raw_affiliation_strings":["Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-0299-9914","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101452320","display_name":"Yifei Wu","orcid":"https://orcid.org/0009-0002-8201-6438"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifei Wu","raw_affiliation_strings":["Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0002-8201-6438","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100341993","display_name":"Yanmin Qian","orcid":"https://orcid.org/0000-0002-0314-3790"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanmin Qian","raw_affiliation_strings":["Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-0314-3790","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":4.2232,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.94889824,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"32","issue":null,"first_page":"1941","last_page":"1953"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8033437728881836},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7538512349128723},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7011510133743286},{"id":"https://openalex.org/keywords/sync","display_name":"sync","score":0.5593713521957397},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.49291324615478516},{"id":"https://openalex.org/keywords/audio-mining","display_name":"Audio mining","score":0.48100408911705017},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4727811813354492},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.47076416015625},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3355298638343811},{"id":"https://openalex.org/keywords/acoustic-model","display_name":"Acoustic model","score":0.3136304020881653},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.25978806614875793},{"id":"https://openalex.org/keywords/frame","display_name":"Frame (networking)","score":0.15004777908325195}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8033437728881836},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7538512349128723},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7011510133743286},{"id":"https://openalex.org/C3913047","wikidata":"https://www.wikidata.org/wiki/Q1956265","display_name":"sync","level":3,"score":0.5593713521957397},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.49291324615478516},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.48100408911705017},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4727811813354492},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.47076416015625},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3355298638343811},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.3136304020881653},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.25978806614875793},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.15004777908325195},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2024.3375641","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3375641","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5465080215","display_name":null,"funder_award_id":"2021SHZDZX0102","funder_id":"https://openalex.org/F4320321885","funder_display_name":"Science and Technology Commission of Shanghai Municipality"}],"funders":[{"id":"https://openalex.org/F4320321885","display_name":"Science and Technology Commission of Shanghai Municipality","ror":"https://ror.org/03kt66j61"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":107,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1503933356","https://openalex.org/W1897240248","https://openalex.org/W1933349210","https://openalex.org/W1993882792","https://openalex.org/W1995562189","https://openalex.org/W2022799064","https://openalex.org/W2035777533","https://openalex.org/W2076462394","https://openalex.org/W2127141656","https://openalex.org/W2143612262","https://openalex.org/W2147768505","https://openalex.org/W2155273149","https://openalex.org/W2160815625","https://openalex.org/W2194775991","https://openalex.org/W2289925289","https://openalex.org/W2327501763","https://openalex.org/W2512544816","https://openalex.org/W2526425061","https://openalex.org/W2551572271","https://openalex.org/W2594690981","https://openalex.org/W2808631503","https://openalex.org/W2889624961","https://openalex.org/W2890952074","https://openalex.org/W2891205112","https://openalex.org/W2892009249","https://openalex.org/W2894280539","https://openalex.org/W2901907199","https://openalex.org/W2936774411","https://openalex.org/W2943845043","https://openalex.org/W2963192057","https://openalex.org/W2963393391","https://openalex.org/W2963528589","https://openalex.org/W2963654155","https://openalex.org/W2963785710","https://openalex.org/W2972541922","https://openalex.org/W2972756321","https://openalex.org/W2973049979","https://openalex.org/W2981087920","https://openalex.org/W2981851019","https://openalex.org/W3006974783","https://openalex.org/W3011234510","https://openalex.org/W3015383493","https://openalex.org/W3015445830","https://openalex.org/W3015579786","https://openalex.org/W3016011581","https://openalex.org/W3024147341","https://openalex.org/W3032514799","https://openalex.org/W3035042697","https://openalex.org/W3094612121","https://openalex.org/W3097450720","https://openalex.org/W3097777922","https://openalex.org/W3102469722","https://openalex.org/W3106423890","https://openalex.org/W3108367559","https://openalex.org/W3110606395","https://openalex.org/W3118120400","https://openalex.org/W3162293946","https://openalex.org/W3170088426","https://openalex.org/W3182657421","https://openalex.org/W3196777808","https://openalex.org/W3198605962","https://openalex.org/W3199527474","https://openalex.org/W3205193540","https://openalex.org/W3205715971","https://openalex.org/W3209059054","https://openalex.org/W3214882941","https://openalex.org/W4221153521","https://openalex.org/W4224920251","https://openalex.org/W4225264578","https://openalex.org/W4287887284","https://openalex.org/W4292793901","https://openalex.org/W4296414218","https://openalex.org/W4297841411","https://openalex.org/W4297841641","https://openalex.org/W4297841719","https://openalex.org/W4298112588","https://openalex.org/W4308236834","https://openalex.org/W4312638101","https://openalex.org/W4312711418","https://openalex.org/W4312852648","https://openalex.org/W4372267507","https://openalex.org/W4372340947","https://openalex.org/W4372346152","https://openalex.org/W4385245566","https://openalex.org/W4385822542","https://openalex.org/W4385822702","https://openalex.org/W4387757537","https://openalex.org/W4389352357","https://openalex.org/W4390190619","https://openalex.org/W4392902926","https://openalex.org/W6623517193","https://openalex.org/W6629930100","https://openalex.org/W6631190155","https://openalex.org/W6638749077","https://openalex.org/W6675365184","https://openalex.org/W6732872814","https://openalex.org/W6754420807","https://openalex.org/W6763608318","https://openalex.org/W6779844698","https://openalex.org/W6780218876","https://openalex.org/W6781364056","https://openalex.org/W6787473416","https://openalex.org/W6791353385","https://openalex.org/W6810168380","https://openalex.org/W6811013733","https://openalex.org/W6847652939"],"related_works":["https://openalex.org/W2547431347","https://openalex.org/W2157598242","https://openalex.org/W2620660273","https://openalex.org/W2619911963","https://openalex.org/W2014474048","https://openalex.org/W2762879233","https://openalex.org/W1969715806","https://openalex.org/W22517275","https://openalex.org/W1554343988","https://openalex.org/W2129389759"],"abstract_inverted_index":{"Audio-Visual":[0],"Speech":[1],"Recognition":[2],"(AVSR)":[3],"is":[4,106],"a":[5,33,63,101,199,221],"promising":[6],"approach":[7,44,135],"to":[8,85,110,146,159,219],"improving":[9],"the":[10,19,46,54,58,87,95,111,128,137,147,160,190,214],"accuracy":[11,119],"and":[12,52,73,79,108,120,155,186,225],"robustness":[13,173],"of":[14,21,97,210],"speech":[15,36],"recognition":[16,37,118],"systems":[17],"with":[18,39,216],"assistance":[20],"visual":[22,182],"cues":[23],"in":[24,57,116,174,203],"challenging":[25,176],"acoustic":[26],"environments.":[27],"In":[28],"this":[29,217],"paper,":[30],"we":[31],"present":[32],"novel":[34],"audio-visual":[35,130,162,167,184],"architecture":[38],"unified":[40,59,98],"cross-modal":[41,99],"attention.":[42],"Our":[43],"concatenates":[45],"sequences":[47],"temporally":[48],"from":[49],"different":[50],"modalities":[51,212],"encodes":[53],"fused":[55],"sequence":[56],"feature":[60],"space":[61],"using":[62],"shared":[64],"Conformer":[65],"encoder.":[66],"We":[67],"then":[68],"explicitly":[69],"model":[70,197,215],"additive":[71],"noise":[72],"potential":[74],"out-of-sync":[75,91],"samples":[76],"during":[77],"training,":[78],"propose":[80],"an":[81],"auxiliary":[82],"asynchronization-aware":[83],"loss":[84],"improve":[86],"system":[88,151,169],"performance":[89],"on":[90,127],"data.":[92],"To":[93],"enhance":[94],"efficacy":[96],"attention,":[100],"manual":[102],"attention":[103],"alignment":[104],"strategy":[105],"designed":[107],"applied":[109],"model,":[112],"bringing":[113],"additional":[114,211],"gains":[115],"both":[117],"computation":[121],"cost.":[122],"As":[123],"demonstrated":[124],"by":[125,142],"experiments":[126],"large-scale":[129],"LRS3":[131],"dataset,":[132],"our":[133],"proposed":[134,166,191],"reduces":[136],"word":[138],"error":[139],"rate":[140],"(WER)":[141],"relatively":[143,156],"50%":[144],"compared":[145,158],"audio-only":[148,180],"single-modal":[149],"ASR":[150,163,168],"under":[152],"noisy":[153],"conditions,":[154,177],"25%":[157],"previous":[161],"baseline.":[164],"The":[165],"also":[170],"shows":[171],"superior":[172],"more":[175,200,222],"such":[178],"as":[179],"data,":[181],"corruption,":[183],"misalignment,":[185],"multi-talker":[187],"interference.":[188],"Moreover,":[189],"<italic":[192],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[193],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Unified":[194],"Cross-Modal":[195],"Attention</i>":[196],"exhibits":[198],"general":[201],"ability":[202],"multi-modality":[204],"fusion,":[205],"allowing":[206],"for":[207],"easy":[208],"integration":[209],"into":[213],"framework":[218],"achieve":[220],"accurate,":[223],"robust,":[224],"safer":[226],"multi-modal":[227],"system.":[228]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}