{"id":"https://openalex.org/W4372267073","doi":"https://doi.org/10.1109/icassp49357.2023.10096726","title":"Speech-Text Based Multi-Modal Training with Bidirectional Attention for Improved Speech Recognition","display_name":"Speech-Text Based Multi-Modal Training with Bidirectional Attention for Improved Speech Recognition","publication_year":2023,"publication_date":"2023-05-05","ids":{"openalex":"https://openalex.org/W4372267073","doi":"https://doi.org/10.1109/icassp49357.2023.10096726"},"language":"en","primary_location":{"id":"doi:10.1109/icassp49357.2023.10096726","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49357.2023.10096726","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011571577","display_name":"Yuhang Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I96908189","display_name":"Xinjiang University","ror":"https://ror.org/059gw8r13","country_code":"CN","type":"education","lineage":["https://openalex.org/I96908189"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuhang Yang","raw_affiliation_strings":["Xinjiang University,School of Information Science and Engineering,Urumqi,China","School of Information Science and Engineering, Xinjiang University, Urumqi, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xinjiang University,School of Information Science and Engineering,Urumqi,China","institution_ids":["https://openalex.org/I96908189"]},{"raw_affiliation_string":"School of Information Science and Engineering, Xinjiang University, Urumqi, China","institution_ids":["https://openalex.org/I96908189"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039696635","display_name":"Haihua Xu","orcid":"https://orcid.org/0000-0002-2220-8465"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haihua Xu","raw_affiliation_strings":["Bytedance"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bytedance","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100757355","display_name":"Hao Huang","orcid":"https://orcid.org/0009-0005-7353-3072"},"institutions":[{"id":"https://openalex.org/I96908189","display_name":"Xinjiang University","ror":"https://ror.org/059gw8r13","country_code":"CN","type":"education","lineage":["https://openalex.org/I96908189"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Huang","raw_affiliation_strings":["Xinjiang University,School of Information Science and Engineering,Urumqi,China","School of Information Science and Engineering, Xinjiang University, Urumqi, China","Xinjiang Provincial Key Laboratory of Multi-Lingual Information Technology, Urumqi, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xinjiang University,School of Information Science and Engineering,Urumqi,China","institution_ids":["https://openalex.org/I96908189"]},{"raw_affiliation_string":"School of Information Science and Engineering, Xinjiang University, Urumqi, China","institution_ids":["https://openalex.org/I96908189"]},{"raw_affiliation_string":"Xinjiang Provincial Key Laboratory of Multi-Lingual Information Technology, Urumqi, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070872826","display_name":"Eng Siong Chng","orcid":"https://orcid.org/0000-0001-6257-7399"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Eng Siong Chng","raw_affiliation_strings":["Nanyang Technological University,School of Computer Science and Engineering,Singapore","School of Computer Science and Engineering, Nanyang Technological University, Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University,School of Computer Science and Engineering,Singapore","institution_ids":["https://openalex.org/I172675005"]},{"raw_affiliation_string":"School of Computer Science and Engineering, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053726259","display_name":"Sheng Li","orcid":"https://orcid.org/0000-0001-7636-3797"},"institutions":[{"id":"https://openalex.org/I90023481","display_name":"National Institute of Information and Communications Technology","ror":"https://ror.org/016bgq349","country_code":"JP","type":"facility","lineage":["https://openalex.org/I90023481"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Sheng Li","raw_affiliation_strings":["National Institute of Information and Communications Technology (NICT),Kyoto,Japan","National Institute of Information and Communications Technology (NICT), Kyoto, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Information and Communications Technology (NICT),Kyoto,Japan","institution_ids":["https://openalex.org/I90023481"]},{"raw_affiliation_string":"National Institute of Information and Communications Technology (NICT), Kyoto, Japan","institution_ids":["https://openalex.org/I90023481"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7144302129745483},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6945875883102417},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6707116961479187},{"id":"https://openalex.org/keywords/word-error-rate","display_name":"Word error rate","score":0.49469372630119324},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4811038374900818},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.46909889578819275},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.42384201288223267},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.4172061085700989},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.09984403848648071}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7144302129745483},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6945875883102417},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6707116961479187},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.49469372630119324},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4811038374900818},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.46909889578819275},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.42384201288223267},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.4172061085700989},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.09984403848648071},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp49357.2023.10096726","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49357.2023.10096726","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5199999809265137,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W2327501763","https://openalex.org/W2577366047","https://openalex.org/W2888779557","https://openalex.org/W2962760690","https://openalex.org/W2962780374","https://openalex.org/W2963739817","https://openalex.org/W2964012862","https://openalex.org/W2964330921","https://openalex.org/W2972943112","https://openalex.org/W2979476256","https://openalex.org/W2982223350","https://openalex.org/W3008370744","https://openalex.org/W3015194534","https://openalex.org/W3015265920","https://openalex.org/W3015280134","https://openalex.org/W3016006013","https://openalex.org/W3016167541","https://openalex.org/W3036601975","https://openalex.org/W3097777922","https://openalex.org/W3162037819","https://openalex.org/W3163605596","https://openalex.org/W3200006799","https://openalex.org/W3200601846","https://openalex.org/W3205644108","https://openalex.org/W3206531472","https://openalex.org/W3209059054","https://openalex.org/W3209371554","https://openalex.org/W3209984917","https://openalex.org/W4221145109","https://openalex.org/W4221156109","https://openalex.org/W4224935349","https://openalex.org/W4225272718","https://openalex.org/W4225295099","https://openalex.org/W4225299129","https://openalex.org/W4226120743","https://openalex.org/W4296069150","https://openalex.org/W6732447497","https://openalex.org/W6743477263","https://openalex.org/W6769196770","https://openalex.org/W6769593479","https://openalex.org/W6775917510","https://openalex.org/W6780218876","https://openalex.org/W6802465204","https://openalex.org/W6803581120","https://openalex.org/W6810007534"],"related_works":["https://openalex.org/W2309273277","https://openalex.org/W2061937230","https://openalex.org/W1574295218","https://openalex.org/W2405439032","https://openalex.org/W2132658536","https://openalex.org/W2070212102","https://openalex.org/W2544241817","https://openalex.org/W2158882055","https://openalex.org/W2914603036","https://openalex.org/W2124470186"],"abstract_inverted_index":{"To":[0,151],"let":[1],"the":[2,28,42,45,90,93,97,113,123,127,139,153,156],"state-of-the-art":[3],"end-to-end":[4],"ASR":[5,68,141],"model":[6],"enjoy":[7],"data":[8,17,149,192,201],"efficiency,":[9],"as":[10,12],"well":[11],"much":[13],"more":[14,119,131,144,198],"unpaired":[15,147,169,199],"text":[16,39,73,128,148,170,200],"by":[18,126],"multi-modal":[19,77],"training,":[20],"one":[21,98],"needs":[22],"to":[23,56,64,83,100,133,182],"address":[24],"two":[25,49,161],"problems:":[26],"1)":[27],"synchronicity":[29],"of":[30,44,92,155,163],"feature":[31,85],"sampling":[32,86],"rates":[33],"between":[34],"speech":[35,114,135],"and":[36,72,137],"language":[37],"(aka":[38],"data);":[40],"2)":[41],"homogeneity":[43],"learned":[46],"representations":[47,115,124],"from":[48],"encoders.":[50],"In":[51],"this":[52],"paper":[53],"we":[54,159],"propose":[55],"employ":[57],"a":[58,76,111],"novel":[59],"bidirectional":[60],"attention":[61],"mechanism":[62],"(BiAM)":[63],"jointly":[65],"learn":[66],"both":[67],"encoder":[69,74,129],"(bottom":[70],"layers)":[71],"with":[75,106,118,165,189],"learning":[78],"method.":[79],"The":[80],"BiAM":[81],"is":[82,202],"facilitate":[84],"rate":[87,186],"exchange,":[88],"realizing":[89],"quality":[91],"transformed":[94],"features":[95],"for":[96,146],"kind":[99],"be":[101],"measured":[102],"in":[103],"another":[104],"space,":[105],"diversified":[107],"objective":[108],"functions.":[109],"As":[110],"result,":[112],"are":[116,130,143],"enriched":[117],"linguistic":[120],"information,":[121],"while":[122,194],"generated":[125],"similar":[132],"corresponding":[134],"ones,":[136],"therefore":[138],"shared":[140],"models":[142],"amenable":[145],"pretraining.":[150],"validate":[152],"efficacy":[154],"proposed":[157],"method,":[158],"perform":[160],"categories":[162],"experiments":[164],"or":[166],"without":[167],"extra":[168],"data.":[171],"Experimental":[172],"results":[173],"on":[174],"Librispeech":[175],"corpus":[176],"show":[177],"it":[178],"can":[179],"achieve":[180],"up":[181],"6.15%":[183],"word":[184],"error":[185],"reduction":[187],"(WERR)":[188],"only":[190],"paired":[191],"learning,":[193],"9.23%":[195],"WERR":[196],"when":[197],"employed":[203],"<sup":[204],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[205],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">1</sup>":[206],".":[207]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}