{"id":"https://openalex.org/W4322577229","doi":"https://doi.org/10.1109/taslp.2023.3244521","title":"Improving Speech Translation by Cross-Modal Multi-Grained Contrastive Learning","display_name":"Improving Speech Translation by Cross-Modal Multi-Grained Contrastive Learning","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4322577229","doi":"https://doi.org/10.1109/taslp.2023.3244521"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2023.3244521","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2023.3244521","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2304.10309","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024433209","display_name":"Hao Zhang","orcid":"https://orcid.org/0000-0001-8852-6311"},"institutions":[{"id":"https://openalex.org/I169689159","display_name":"PLA Information Engineering University","ror":"https://ror.org/00mm1qk40","country_code":"CN","type":"education","lineage":["https://openalex.org/I169689159"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Zhang","raw_affiliation_strings":["School of the Information Engineering University, Zhengzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-8852-6311","affiliations":[{"raw_affiliation_string":"School of the Information Engineering University, Zhengzhou, China","institution_ids":["https://openalex.org/I169689159"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024319575","display_name":"Nianwen Si","orcid":"https://orcid.org/0000-0003-4619-4325"},"institutions":[{"id":"https://openalex.org/I169689159","display_name":"PLA Information Engineering University","ror":"https://ror.org/00mm1qk40","country_code":"CN","type":"education","lineage":["https://openalex.org/I169689159"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nianwen Si","raw_affiliation_strings":["School of the Information Engineering University, Zhengzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of the Information Engineering University, Zhengzhou, China","institution_ids":["https://openalex.org/I169689159"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101560152","display_name":"Yaqi Chen","orcid":"https://orcid.org/0000-0001-6301-802X"},"institutions":[{"id":"https://openalex.org/I169689159","display_name":"PLA Information Engineering University","ror":"https://ror.org/00mm1qk40","country_code":"CN","type":"education","lineage":["https://openalex.org/I169689159"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaqi Chen","raw_affiliation_strings":["School of the Information Engineering University, Zhengzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-6301-802X","affiliations":[{"raw_affiliation_string":"School of the Information Engineering University, Zhengzhou, China","institution_ids":["https://openalex.org/I169689159"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100635256","display_name":"Wenlin Zhang","orcid":"https://orcid.org/0000-0002-6842-9722"},"institutions":[{"id":"https://openalex.org/I169689159","display_name":"PLA Information Engineering University","ror":"https://ror.org/00mm1qk40","country_code":"CN","type":"education","lineage":["https://openalex.org/I169689159"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenlin Zhang","raw_affiliation_strings":["Department of Electronic Engineering, Tsinghua University, Beijing, China","School of the Information Engineering University, Zhengzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"School of the Information Engineering University, Zhengzhou, China","institution_ids":["https://openalex.org/I169689159"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074428051","display_name":"Xukui Yang","orcid":"https://orcid.org/0000-0002-7989-7089"},"institutions":[{"id":"https://openalex.org/I169689159","display_name":"PLA Information Engineering University","ror":"https://ror.org/00mm1qk40","country_code":"CN","type":"education","lineage":["https://openalex.org/I169689159"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xukui Yang","raw_affiliation_strings":["School of the Information Engineering University, Zhengzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-7989-7089","affiliations":[{"raw_affiliation_string":"School of the Information Engineering University, Zhengzhou, China","institution_ids":["https://openalex.org/I169689159"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001488518","display_name":"Dan Qu","orcid":"https://orcid.org/0000-0001-9917-7794"},"institutions":[{"id":"https://openalex.org/I169689159","display_name":"PLA Information Engineering University","ror":"https://ror.org/00mm1qk40","country_code":"CN","type":"education","lineage":["https://openalex.org/I169689159"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dan Qu","raw_affiliation_strings":["School of the Information Engineering University, Zhengzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-9917-7794","affiliations":[{"raw_affiliation_string":"School of the Information Engineering University, Zhengzhou, China","institution_ids":["https://openalex.org/I169689159"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100692904","display_name":"Wei-Qiang Zhang","orcid":"https://orcid.org/0000-0003-3841-1959"},"institutions":[{"id":"https://openalex.org/I169689159","display_name":"PLA Information Engineering University","ror":"https://ror.org/00mm1qk40","country_code":"CN","type":"education","lineage":["https://openalex.org/I169689159"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei-Qiang Zhang","raw_affiliation_strings":["Department of Electronic Engineering, Tsinghua University, Beijing, China","School of the Information Engineering University, Zhengzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-3841-1959","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"School of the Information Engineering University, Zhengzhou, China","institution_ids":["https://openalex.org/I169689159"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.095,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.89056806,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"31","issue":null,"first_page":"1075","last_page":"1086"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5566514134407043},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5502076148986816},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5199000835418701}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5566514134407043},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5502076148986816},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5199000835418701}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/taslp.2023.3244521","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2023.3244521","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2304.10309","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10309","pdf_url":"https://arxiv.org/pdf/2304.10309","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2304.10309","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10309","pdf_url":"https://arxiv.org/pdf/2304.10309","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.6899999976158142,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G1014351457","display_name":null,"funder_award_id":"62276153","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7804056170","display_name":null,"funder_award_id":"62171470","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4322577229.pdf"},"referenced_works_count":77,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W2113106066","https://openalex.org/W2134546430","https://openalex.org/W2187089797","https://openalex.org/W2407080277","https://openalex.org/W2466918907","https://openalex.org/W2582956876","https://openalex.org/W2605131327","https://openalex.org/W2936774411","https://openalex.org/W2936969148","https://openalex.org/W2962784628","https://openalex.org/W2964161387","https://openalex.org/W2964172053","https://openalex.org/W2972448360","https://openalex.org/W2998386507","https://openalex.org/W3001434439","https://openalex.org/W3006988520","https://openalex.org/W3015633994","https://openalex.org/W3034571331","https://openalex.org/W3034999214","https://openalex.org/W3035524453","https://openalex.org/W3036601975","https://openalex.org/W3037217258","https://openalex.org/W3037309139","https://openalex.org/W3096490862","https://openalex.org/W3097301532","https://openalex.org/W3102811925","https://openalex.org/W3105669983","https://openalex.org/W3105816068","https://openalex.org/W3113908264","https://openalex.org/W3134486096","https://openalex.org/W3139958517","https://openalex.org/W3154229486","https://openalex.org/W3156636935","https://openalex.org/W3161101519","https://openalex.org/W3168822201","https://openalex.org/W3175362188","https://openalex.org/W3176382501","https://openalex.org/W3176455679","https://openalex.org/W3176711365","https://openalex.org/W3196833881","https://openalex.org/W3199681947","https://openalex.org/W4206468798","https://openalex.org/W4221151575","https://openalex.org/W4221163209","https://openalex.org/W4223622550","https://openalex.org/W4226212120","https://openalex.org/W4226499556","https://openalex.org/W4284898017","https://openalex.org/W4285105102","https://openalex.org/W4285241989","https://openalex.org/W4286685063","https://openalex.org/W4287890956","https://openalex.org/W4300558631","https://openalex.org/W4309872123","https://openalex.org/W4385245566","https://openalex.org/W4385573012","https://openalex.org/W6631190155","https://openalex.org/W6745682157","https://openalex.org/W6748215858","https://openalex.org/W6750200984","https://openalex.org/W6752726010","https://openalex.org/W6757423643","https://openalex.org/W6758113384","https://openalex.org/W6769269733","https://openalex.org/W6780218876","https://openalex.org/W6780294235","https://openalex.org/W6785130102","https://openalex.org/W6791353385","https://openalex.org/W6791429434","https://openalex.org/W6794146050","https://openalex.org/W6801971945","https://openalex.org/W6804095316","https://openalex.org/W6810362114","https://openalex.org/W6811255430","https://openalex.org/W6846397321","https://openalex.org/W6898634591"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W4391913857","https://openalex.org/W2350741829","https://openalex.org/W3204019825"],"abstract_inverted_index":{"The":[0,39],"end-to-end":[1],"speech":[2,152],"translation":[3,55],"(E2E-ST)":[4],"model":[5,29,47,80,91],"has":[6],"gradually":[7],"become":[8],"a":[9,28,162],"mainstream":[10],"paradigm":[11],"due":[12,31],"to":[13,25,32,51,69,145,166,220],"its":[14,209],"low":[15],"latency":[16],"and":[17,36,143,216],"less":[18],"error":[19],"propagation.":[20],"However,":[21,77],"it":[22],"is":[23,104,136],"non-trivial":[24],"train":[26],"such":[27],"well":[30],"the":[33,45,52,60,78,85,89,97,114,147,168,172,182,192],"task":[34,87],"complexity":[35],"data":[37],"scarcity.":[38],"speech-and-text":[40],"modality":[41],"differences":[42],"result":[43],"in":[44,171],"E2E-ST":[46,194],"performance":[48],"usually":[49],"inferior":[50],"corresponding":[53],"machine":[54],"(MT)":[56],"model.":[57],"Based":[58],"on":[59,84,181,196],"above":[61],"observation,":[62],"existing":[63],"methods":[64],"often":[65,81],"use":[66],"sharing":[67],"mechanisms":[68],"carry":[70],"outimplicit":[71],"knowledge":[72,98,124],"transferby":[73],"imposing":[74],"various":[75],"constraints.":[76],"final":[79],"performs":[82],"worse":[83],"MT":[86,90,173],"than":[88],"trained":[92],"alone,":[93],"which":[94,122,175],"means":[95],"that":[96,186,204],"transfer":[99],"ability":[100],"of":[101,133],"this":[102],"method":[103,165],"also":[105],"limited.":[106],"To":[107],"deal":[108],"with":[109],"these":[110],"problems,":[111],"we":[112,160],"propose":[113],"FCCL":[115,205],"(Fine-":[116],"andCoarse-":[117],"GranularityContrastiveLearning)":[118],"approach":[119,135,189],"for":[120,150],"E2E-ST,":[121],"makesexplicit":[123],"transferthrough":[125],"cross-modal":[126],"multi-grained":[127],"contrastive":[128,138],"learning.":[129,179],"A":[130],"key":[131],"ingredient":[132],"our":[134,187],"applying":[137],"learning":[139,212],"at":[140],"both":[141],"sentence-":[142],"frame-level":[144],"give":[146],"comprehensive":[148],"guide":[149],"extracting":[151],"representations":[153],"containing":[154],"rich":[155],"semantic":[156,222],"information.":[157,223],"In":[158],"addition,":[159],"adopt":[161],"simple":[163],"whitening":[164],"alleviate":[167],"representation":[169],"degeneration":[170],"model,":[174],"adversely":[176],"affects":[177],"contrast":[178],"Experiments":[180],"MuST-C":[183],"benchmark":[184],"show":[185],"proposed":[188],"significantly":[190],"outperforms":[191],"state-of-the-art":[193],"baselines":[195],"all":[197],"eight":[198],"language":[199],"pairs.":[200],"Further":[201],"analysis":[202],"indicates":[203],"can":[206],"free":[207],"up":[208],"capacity":[210],"from":[211],"grammatical":[213],"structure":[214],"information":[215],"force":[217],"more":[218],"layers":[219],"learn":[221]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":5}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}