{"id":"https://openalex.org/W4394862800","doi":"https://doi.org/10.1109/taslp.2024.3389630","title":"Conversational Speech Recognition by Learning Audio-Textual Cross-Modal Contextual Representation","display_name":"Conversational Speech Recognition by Learning Audio-Textual Cross-Modal Contextual Representation","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4394862800","doi":"https://doi.org/10.1109/taslp.2024.3389630"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2024.3389630","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3389630","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100649112","display_name":"Kun Wei","orcid":null},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Wei","raw_affiliation_strings":["Audio, Speech and Language Processing Group, School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-1480-9009","affiliations":[{"raw_affiliation_string":"Audio, Speech and Language Processing Group, School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107028150","display_name":"Bei Li","orcid":"https://orcid.org/0000-0001-7617-9041"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bei Li","raw_affiliation_strings":["School of Computer Science and Engineering, Northeastern University, Shenyang, China"],"raw_orcid":"https://orcid.org/0000-0001-7617-9041","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Northeastern University, Shenyang, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100751610","display_name":"Hang Lv","orcid":"https://orcid.org/0000-0003-3761-1684"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hang Lv","raw_affiliation_strings":["Audio, Speech and Language Processing Group, School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0003-3761-1684","affiliations":[{"raw_affiliation_string":"Audio, Speech and Language Processing Group, School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104212958","display_name":"Quan L\u00fc","orcid":null},"institutions":[{"id":"https://openalex.org/I4210092002","display_name":"Runze (China)","ror":"https://ror.org/00fgwkr80","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210092002"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Quan Lu","raw_affiliation_strings":["Mashang Consumer Finance Company, Ltd., Chongqing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mashang Consumer Finance Company, Ltd., Chongqing, China","institution_ids":["https://openalex.org/I4210092002"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101654008","display_name":"Ning Jiang","orcid":"https://orcid.org/0000-0003-4549-6372"},"institutions":[{"id":"https://openalex.org/I4210092002","display_name":"Runze (China)","ror":"https://ror.org/00fgwkr80","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210092002"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ning Jiang","raw_affiliation_strings":["Mashang Consumer Finance Company, Ltd., Chongqing, China"],"raw_orcid":"https://orcid.org/0000-0003-4549-6372","affiliations":[{"raw_affiliation_string":"Mashang Consumer Finance Company, Ltd., Chongqing, China","institution_ids":["https://openalex.org/I4210092002"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100668966","display_name":"Lei Xie","orcid":"https://orcid.org/0000-0001-8234-0823"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Xie","raw_affiliation_strings":["Audio, Speech and Language Processing Group, School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-8234-0823","affiliations":[{"raw_affiliation_string":"Audio, Speech and Language Processing Group, School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6671,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.85557062,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"32","issue":null,"first_page":"2432","last_page":"2444"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.805924117565155},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6725864410400391},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.6235454082489014},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6100590229034424},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.544683575630188},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.4909699261188507},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4806046187877655},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4753432273864746},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4713183045387268},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.45594322681427},{"id":"https://openalex.org/keywords/coherence","display_name":"Coherence (philosophical gambling strategy)","score":0.41954949498176575},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.13512879610061646}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.805924117565155},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6725864410400391},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.6235454082489014},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6100590229034424},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.544683575630188},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.4909699261188507},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4806046187877655},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4753432273864746},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4713183045387268},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.45594322681427},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.41954949498176575},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.13512879610061646},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2024.3389630","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3389630","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W108866686","https://openalex.org/W179875071","https://openalex.org/W1526236009","https://openalex.org/W1999965501","https://openalex.org/W2144499799","https://openalex.org/W2160815625","https://openalex.org/W2526425061","https://openalex.org/W2759071281","https://openalex.org/W2886025712","https://openalex.org/W2891176389","https://openalex.org/W2892009249","https://openalex.org/W2896457183","https://openalex.org/W2933138175","https://openalex.org/W2936774411","https://openalex.org/W2952370363","https://openalex.org/W2962780374","https://openalex.org/W2963714898","https://openalex.org/W2964110616","https://openalex.org/W2964669873","https://openalex.org/W2965373594","https://openalex.org/W2972430654","https://openalex.org/W3096798607","https://openalex.org/W3097777922","https://openalex.org/W3163463193","https://openalex.org/W3173680274","https://openalex.org/W3173978205","https://openalex.org/W3197580070","https://openalex.org/W3197898596","https://openalex.org/W3203407300","https://openalex.org/W3207222250","https://openalex.org/W3209059054","https://openalex.org/W3211278025","https://openalex.org/W4221144554","https://openalex.org/W4221145109","https://openalex.org/W4283826357","https://openalex.org/W4283835659","https://openalex.org/W4297841248","https://openalex.org/W4297841547","https://openalex.org/W4310271076","https://openalex.org/W4311724836","https://openalex.org/W4320930577","https://openalex.org/W4372260432","https://openalex.org/W4372260611","https://openalex.org/W4372341362","https://openalex.org/W4385245566","https://openalex.org/W4385822425","https://openalex.org/W4385823039","https://openalex.org/W4392979802","https://openalex.org/W6687045409","https://openalex.org/W6731370813","https://openalex.org/W6746023985","https://openalex.org/W6766673545","https://openalex.org/W6780218876","https://openalex.org/W6797307915","https://openalex.org/W6803092890","https://openalex.org/W6810007534","https://openalex.org/W6846807982","https://openalex.org/W6847032292"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W1968552888","https://openalex.org/W2374116601","https://openalex.org/W3093134843","https://openalex.org/W1511346092","https://openalex.org/W1527532029","https://openalex.org/W2378167147","https://openalex.org/W3210777354","https://openalex.org/W2281307425","https://openalex.org/W2772323916"],"abstract_inverted_index":{"Automatic":[0],"Speech":[1],"Recognition":[2],"(ASR)":[3],"in":[4],"conversational":[5,17,45,55,115],"settings":[6],"presents":[7],"unique":[8],"challenges,":[9],"including":[10],"extracting":[11],"relevant":[12],"contextual":[13],"information":[14,126],"from":[15],"previous":[16],"turns.":[18],"Due":[19],"to":[20,30,99,145],"irrelevant":[21],"content,":[22],"error":[23,90],"propagation,":[24],"and":[25,33,67,74,107,114,134,141],"redundancy,":[26],"existing":[27],"methods":[28],"struggle":[29],"extract":[31],"longer":[32,123],"more":[34],"effective":[35],"contexts.":[36],"To":[37],"address":[38],"this":[39],"issue,":[40],"we":[41],"introduce":[42],"a":[43,60,71,75],"novel":[44],"ASR":[46],"system,":[47],"extending":[48],"the":[49,81,118,146],"Conformer":[50,148],"encoder-decoder":[51],"model":[52,121],"with":[53],"cross-modal":[54,61,113],"representation.":[56],"Our":[57],"approach":[58],"leverages":[59],"extractor":[62],"that":[63],"combines":[64],"pre-trained":[65],"speech":[66,86],"text":[68],"models":[69],"through":[70],"specialized":[72],"encoder":[73],"modal-level":[76],"mask":[77],"input.":[78],"This":[79],"enables":[80],"extraction":[82],"of":[83,132],"richer":[84],"historical":[85],"context":[87,124],"without":[88,125],"explicit":[89],"propagation.":[91],"We":[92],"also":[93],"incorporate":[94],"conditional":[95],"latent":[96],"variational":[97],"modules":[98],"learn":[100],"conversational-level":[101],"attributes":[102],"such":[103],"as":[104],"role":[105],"preference":[106],"topic":[108],"coherence.":[109],"By":[110],"introducing":[111],"both":[112],"representations":[116],"into":[117],"decoder,":[119],"our":[120],"retains":[122],"loss,":[127],"achieving":[128],"relative":[129],"accuracy":[130],"improvements":[131],"8.8%":[133],"23%":[135],"on":[136],"Mandarin":[137],"conversation":[138],"datasets":[139],"HKUST":[140],"MagicData-RAMC,":[142],"respectively,":[143],"compared":[144],"standard":[147],"model.":[149]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2024-04-17T00:00:00"}