{"id":"https://openalex.org/W4416059734","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1004","title":"S2ST-Omni: Hierarchical Language-Aware SpeechLLM Adaptation for Multilingual Speech-to-Speech Translation","display_name":"S2ST-Omni: Hierarchical Language-Aware SpeechLLM Adaptation for Multilingual Speech-to-Speech Translation","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W4416059734","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1004"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.1004","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1004","pdf_url":"https://aclanthology.org/2026.findings-acl.1004.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-acl.1004.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028107330","display_name":"Yu Pan","orcid":"https://orcid.org/0009-0000-4995-0689"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu Pan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090578536","display_name":"X . Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiongfei Wu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100402382","display_name":"Yuguang Yang","orcid":"https://orcid.org/0000-0003-1353-1815"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang Yuguang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088838965","display_name":"Jianhao Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jixun Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000695937","display_name":"Maxime Cordy","orcid":"https://orcid.org/0000-0001-8312-1358"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maxime Cordy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064304208","display_name":"Lei Ma","orcid":"https://orcid.org/0000-0002-0906-3978"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lei Ma","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128504066","display_name":"J M Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jianjun Zhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02077034,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"20114","last_page":"20124"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.49549999833106995,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.49549999833106995,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2921000123023987,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.027899999171495438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adapter","display_name":"Adapter (computing)","score":0.6579999923706055},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.6015999913215637},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.5870000123977661},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.4934000074863434},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.4034000039100647},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.38769999146461487}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8210999965667725},{"id":"https://openalex.org/C177284502","wikidata":"https://www.wikidata.org/wiki/Q1005390","display_name":"Adapter (computing)","level":2,"score":0.6579999923706055},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.6015999913215637},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5870000123977661},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.4934000074863434},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48660001158714294},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4214000105857849},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.4034000039100647},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38960000872612},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.38769999146461487},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.34220001101493835},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.3353999853134155},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.334199994802475},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.3301999866962433},{"id":"https://openalex.org/C53893814","wikidata":"https://www.wikidata.org/wiki/Q7378909","display_name":"Rule-based machine translation","level":2,"score":0.2685999870300293}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.1004","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1004","pdf_url":"https://aclanthology.org/2026.findings-acl.1004.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2506.11160","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.11160","pdf_url":"https://arxiv.org/pdf/2506.11160","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2506.11160","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.11160","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.1004","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1004","pdf_url":"https://aclanthology.org/2026.findings-acl.1004.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416059734.pdf","grobid_xml":"https://content.openalex.org/works/W4416059734.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Despite":[0],"recent":[1,208],"advances":[2],"in":[3,138],"speech-to-speech":[4],"translation":[5,14,33,47,136,159],"(S2ST),":[6],"it":[7],"remains":[8,170],"difficult":[9],"to":[10,63,117,203],"achieve":[11],"both":[12],"high":[13],"accuracy":[15],"and":[16,48,69,105,156,173,196,201],"practical":[17],"flexibility.In":[18],"this":[19,78,165],"paper,":[20],"we":[21,53,80,144],"present":[22],"S2ST-Omni,":[23],"a":[24,30,37,55,60,73,83,111,146],"compositional":[25],"S2ST":[26,209],"framework":[27],"that":[28,58,87,133,151,189],"integrates":[29],"high-accuracy":[31],"speech-to-text":[32],"(S2TT)":[34],"frontend":[35],"with":[36,110,177],"modular,":[38],"plug-andplay":[39],"text-to-speech":[40],"(TTS)":[41],"backend,":[42],"enabling":[43],"independent":[44],"optimization":[45],"of":[46,164],"synthesis.On":[49],"the":[50,65,95,115,124,139,183,193],"S2TT":[51,184],"side,":[52],"introduce":[54],"hybrid":[56],"adapter":[57,103],"follows":[59],"\"local-then-global\"":[61],"strategy":[62,150],"bridge":[64],"pretrained":[66],"Whisper":[67],"encoder":[68],"Qwen3":[70],"LLM,":[71],"yielding":[72],"hierarchical":[74,84],"acoustic-to-semantic":[75],"abstraction.Building":[76],"on":[77,101,162,186],"bridge,":[79],"further":[81],"propose":[82],"language-aware":[85],"architecture":[86],"injects":[88],"sourcelanguage":[89],"information":[90],"at":[91],"two":[92],"complementary":[93],"levels.At":[94],"acoustic":[96,122],"level,":[97,126],"Language-Aware":[98,127],"Dual-CTC":[99],"operates":[100],"intermediate":[102],"features":[104],"employs":[106],"FiLM-style":[107],"feature":[108],"modulation":[109],"learnable":[112],"gate,":[113],"encouraging":[114],"model":[116],"learn":[118],"language-specific":[119,135],"but":[120],"contentfaithful":[121],"representations.At":[123],"linguistic":[125],"Prompting":[128],"dynamically":[129],"constructs":[130],"source-language-conditioned":[131],"prompts":[132],"activate":[134],"knowledge":[137],"LLM.To":[140],"enable":[141],"efficient":[142],"optimization,":[143],"design":[145],"task-specific":[147],"progressive":[148],"fine-tuning":[149],"first":[152],"stabilizes":[153],"speech-text":[154],"alignment":[155],"then":[157],"improves":[158],"via":[160],"LoRA":[161],"top":[163],"converged":[166],"foundation.The":[167],"TTS":[168],"backend":[169],"fully":[171],"modular":[172],"can":[174],"be":[175],"instantiated":[176],"any":[178],"stateof-the-art":[179],"synthesizer":[180],"without":[181],"retraining":[182],"frontend.Experiments":[185],"CVSS-C":[187],"show":[188],"S2ST-Omni":[190],"consistently":[191],"achieves":[192],"best":[194],"BLEU":[195],"ASR-BLEU":[197],"across":[198],"French,":[199],"German,":[200],"Spanish":[202],"English":[204],"directions,":[205],"outperforming":[206],"strong":[207],"baselines.":[210]},"counts_by_year":[],"updated_date":"2026-08-06T08:24:18.245995","created_date":"2025-10-10T00:00:00"}