{"id":"https://openalex.org/W4392902778","doi":"https://doi.org/10.1109/icassp48485.2024.10447926","title":"Cross-Modal Multi-Tasking for Speech-to-Text Translation via Hard Parameter Sharing","display_name":"Cross-Modal Multi-Tasking for Speech-to-Text Translation via Hard Parameter Sharing","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392902778","doi":"https://doi.org/10.1109/icassp48485.2024.10447926"},"language":"en","primary_location":{"id":"doi:10.1109/icassp48485.2024.10447926","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447926","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021201726","display_name":"Brian Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian Yan","raw_affiliation_strings":["Carnegie Mellon University,US","Carnegie Mellon University, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,US","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, US","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050058892","display_name":"Xuankai Chang","orcid":"https://orcid.org/0000-0002-5221-5412"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xuankai Chang","raw_affiliation_strings":["Carnegie Mellon University,US","Carnegie Mellon University, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,US","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, US","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013793053","display_name":"Antonios Anastasopoulos","orcid":"https://orcid.org/0000-0002-8544-246X"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Antonios Anastasopoulos","raw_affiliation_strings":["George Mason University,US","George Mason University, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"George Mason University,US","institution_ids":["https://openalex.org/I162714631"]},{"raw_affiliation_string":"George Mason University, US","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040732498","display_name":"Yuya Fujita","orcid":"https://orcid.org/0000-0001-8155-6040"},"institutions":[{"id":"https://openalex.org/I1325784139","display_name":"Yahoo (United Kingdom)","ror":"https://ror.org/038p3gq39","country_code":"GB","type":"company","lineage":["https://openalex.org/I1325784139","https://openalex.org/I4210134091"]},{"id":"https://openalex.org/I2800095910","display_name":"Yahoo (Spain)","ror":"https://ror.org/03gq8sg42","country_code":"ES","type":"company","lineage":["https://openalex.org/I2800095910","https://openalex.org/I4210134091"]},{"id":"https://openalex.org/I4210134091","display_name":"Yahoo (United States)","ror":"https://ror.org/040dkzz12","country_code":"US","type":"company","lineage":["https://openalex.org/I4210134091"]}],"countries":["ES","GB","US"],"is_corresponding":false,"raw_author_name":"Yuya Fujita","raw_affiliation_strings":["Yahoo Japan Corporation,JP","Yahoo Japan Corporation, JP"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yahoo Japan Corporation,JP","institution_ids":["https://openalex.org/I1325784139","https://openalex.org/I4210134091","https://openalex.org/I2800095910"]},{"raw_affiliation_string":"Yahoo Japan Corporation, JP","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001291873","display_name":"Shinji Watanabe","orcid":"https://orcid.org/0000-0002-5970-8631"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shinji Watanabe","raw_affiliation_strings":["Carnegie Mellon University,US","Carnegie Mellon University, US","Johns Hopkins University, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,US","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, US","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Johns Hopkins University, US","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"11941","last_page":"11945"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8450956344604492},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.6994344592094421},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6333914399147034},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.590314507484436},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.562566339969635},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5056002140045166},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.49548113346099854},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.47751086950302124},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.46942073106765747},{"id":"https://openalex.org/keywords/connectionism","display_name":"Connectionism","score":0.41096335649490356},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.21932771801948547}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8450956344604492},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.6994344592094421},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6333914399147034},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.590314507484436},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.562566339969635},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5056002140045166},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.49548113346099854},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.47751086950302124},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.46942073106765747},{"id":"https://openalex.org/C8521452","wikidata":"https://www.wikidata.org/wiki/Q203790","display_name":"Connectionism","level":3,"score":0.41096335649490356},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.21932771801948547},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp48485.2024.10447926","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp48485.2024.10447926","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320307110","display_name":"Delta","ror":"https://ror.org/03g9c1e75"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":65,"referenced_works":["https://openalex.org/W2127141656","https://openalex.org/W2327501763","https://openalex.org/W2512924740","https://openalex.org/W2624871570","https://openalex.org/W2752796333","https://openalex.org/W2758375579","https://openalex.org/W2766219058","https://openalex.org/W2936774411","https://openalex.org/W2963250244","https://openalex.org/W2963532001","https://openalex.org/W2964012862","https://openalex.org/W3001434439","https://openalex.org/W3024605872","https://openalex.org/W3035207248","https://openalex.org/W3037217258","https://openalex.org/W3046368065","https://openalex.org/W3089472875","https://openalex.org/W3153583341","https://openalex.org/W3168212167","https://openalex.org/W3173767661","https://openalex.org/W3180374548","https://openalex.org/W3205644108","https://openalex.org/W3209059054","https://openalex.org/W3209984917","https://openalex.org/W3215615641","https://openalex.org/W4221153524","https://openalex.org/W4221155340","https://openalex.org/W4221163209","https://openalex.org/W4223622550","https://openalex.org/W4225308107","https://openalex.org/W4226120743","https://openalex.org/W4285223043","https://openalex.org/W4287890956","https://openalex.org/W4307323391","https://openalex.org/W4313679638","https://openalex.org/W4319862255","https://openalex.org/W4372349107","https://openalex.org/W4375869259","https://openalex.org/W4377865046","https://openalex.org/W4381827575","https://openalex.org/W4385565440","https://openalex.org/W4385570550","https://openalex.org/W4385570757","https://openalex.org/W4385571303","https://openalex.org/W4385822683","https://openalex.org/W4386566728","https://openalex.org/W4388017359","https://openalex.org/W4391021666","https://openalex.org/W4394671563","https://openalex.org/W6638749077","https://openalex.org/W6679434410","https://openalex.org/W6739365718","https://openalex.org/W6769269733","https://openalex.org/W6777028661","https://openalex.org/W6781811055","https://openalex.org/W6790356757","https://openalex.org/W6798098866","https://openalex.org/W6810259195","https://openalex.org/W6838475196","https://openalex.org/W6839510803","https://openalex.org/W6846505686","https://openalex.org/W6848735303","https://openalex.org/W6852781825","https://openalex.org/W6853998256","https://openalex.org/W6898634591"],"related_works":["https://openalex.org/W4205841273","https://openalex.org/W4205525690","https://openalex.org/W1761388607","https://openalex.org/W1997922073","https://openalex.org/W2604685715","https://openalex.org/W2412160900","https://openalex.org/W2136453575","https://openalex.org/W2949968076","https://openalex.org/W2935811960","https://openalex.org/W2969407538"],"abstract_inverted_index":{"Recent":[0],"works":[1],"in":[2,47],"end-to-end":[3],"speech-to-text":[4],"translation":[5,18],"(ST)":[6],"have":[7],"proposed":[8],"multi-tasking":[9,41,102],"methods":[10],"with":[11,43],"soft":[12],"parameter":[13,45],"sharing":[14,46],"which":[15,48,66],"leverage":[16],"machine":[17],"(MT)":[19],"data":[20],"via":[21,62],"secondary":[22],"encoders":[23],"that":[24,100,130],"map":[25],"text":[26,70],"inputs":[27,71],"to":[28,84],"an":[29,117],"eventual":[30],"cross-modal":[31],"representation.":[32],"In":[33],"this":[34,81,131],"work,":[35],"we":[36,98,128],"instead":[37],"propose":[38],"a":[39,63,91],"ST/MT":[40],"framework":[42,103,132],"hard":[44],"all":[49],"model":[50],"parameters":[51],"are":[52],"shared":[53],"cross-modally.":[54],"Our":[55],"method":[56],"reduces":[57],"the":[58],"speech-text":[59],"modality":[60],"gap":[61],"pre-processing":[64],"stage":[65],"converts":[67],"speech":[68],"and":[69,112,140],"into":[72],"two":[73],"discrete":[74],"token":[75],"sequences":[76],"of":[77,119],"similar":[78],"length":[79],"\u2013":[80],"allows":[82],"models":[83,115],"indiscriminately":[85],"process":[86],"both":[87],"modalities":[88],"simply":[89],"using":[90],"joint":[92,113],"vocabulary.":[93],"With":[94],"experiments":[95],"on":[96],"MuST-C,":[97],"demonstrate":[99],"our":[101],"improves":[104,142],"attentional":[105],"encoder-decoder,":[106],"Connectionist":[107],"Temporal":[108],"Classification":[109],"(CTC),":[110],"transducer,":[111],"CTC/attention":[114],"by":[116],"average":[118],"+0.5":[120],"BLEU":[121],"without":[122],"any":[123],"external":[124,134],"MT":[125,135],"data.":[126],"Further,":[127],"show":[129],"incorporates":[133],"data,":[136],"yielding":[137,149],"+0.8":[138],"BLEU,":[139],"also":[141],"transfer":[143],"learning":[144],"from":[145],"pre-trained":[146],"textual":[147],"models,":[148],"+1.8":[150],"BLEU.":[151],"<sup":[152],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[153],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">1</sup>":[154]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}