{"id":"https://openalex.org/W4400111385","doi":"https://doi.org/10.1109/taslp.2024.3419418","title":"SpeechX: Neural Codec Language Model as a Versatile Speech Transformer","display_name":"SpeechX: Neural Codec Language Model as a Versatile Speech Transformer","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4400111385","doi":"https://doi.org/10.1109/taslp.2024.3419418"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2024.3419418","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3419418","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063074341","display_name":"Xiaofei Wang","orcid":"https://orcid.org/0000-0002-8837-401X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaofei Wang","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8837-401X","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028363114","display_name":"Manthan Thakker","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Manthan Thakker","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100345092","display_name":"Zhuo Chen","orcid":"https://orcid.org/0000-0002-9011-7928"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhuo Chen","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016279564","display_name":"Naoyuki Kanda","orcid":"https://orcid.org/0000-0002-8628-3288"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Naoyuki Kanda","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8628-3288","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026088950","display_name":"\u015eefik Emre Eskimez","orcid":"https://orcid.org/0000-0001-6259-5925"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sefik Emre Eskimez","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079533447","display_name":"Sanyuan Chen","orcid":"https://orcid.org/0000-0002-3082-6052"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sanyuan Chen","raw_affiliation_strings":["Microsoft Research Asia, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3082-6052","affiliations":[{"raw_affiliation_string":"Microsoft Research Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106627865","display_name":"Min Tang","orcid":"https://orcid.org/0009-0006-3428-9967"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Min Tang","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0006-3428-9967","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101635405","display_name":"Shujie Liu","orcid":"https://orcid.org/0009-0008-2599-6752"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shujie Liu","raw_affiliation_strings":["Microsoft Research Asia, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100365053","display_name":"Jinyu Li","orcid":"https://orcid.org/0000-0002-1089-9748"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jinyu Li","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1089-9748","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101618071","display_name":"Takuya Yoshioka","orcid":"https://orcid.org/0009-0003-7791-3545"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Takuya Yoshioka","raw_affiliation_strings":["Microsoft Corporation, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0003-7791-3545","affiliations":[{"raw_affiliation_string":"Microsoft Corporation, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.7246,"has_fulltext":false,"cited_by_count":35,"citation_normalized_percentile":{"value":0.98449338,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"32","issue":null,"first_page":"3355","last_page":"3364"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9307000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9307000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6752513647079468},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6713560819625854},{"id":"https://openalex.org/keywords/adaptive-multi-rate-audio-codec","display_name":"Adaptive Multi-Rate audio codec","score":0.4974720776081085},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4870748817920685},{"id":"https://openalex.org/keywords/codec","display_name":"Codec","score":0.4386309087276459},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.34931832551956177},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3229343295097351},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.2193414270877838},{"id":"https://openalex.org/keywords/voice-activity-detection","display_name":"Voice activity detection","score":0.16329005360603333},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.12445449829101562},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11882424354553223},{"id":"https://openalex.org/keywords/computer-hardware","display_name":"Computer hardware","score":0.10061535239219666},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.057598501443862915}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6752513647079468},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6713560819625854},{"id":"https://openalex.org/C177067256","wikidata":"https://www.wikidata.org/wiki/Q4676210","display_name":"Adaptive Multi-Rate audio codec","level":4,"score":0.4974720776081085},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4870748817920685},{"id":"https://openalex.org/C161765866","wikidata":"https://www.wikidata.org/wiki/Q184748","display_name":"Codec","level":2,"score":0.4386309087276459},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34931832551956177},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3229343295097351},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.2193414270877838},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.16329005360603333},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.12445449829101562},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11882424354553223},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.10061535239219666},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.057598501443862915}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2024.3419418","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3419418","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W1552314771","https://openalex.org/W2219249508","https://openalex.org/W2408688265","https://openalex.org/W2951130829","https://openalex.org/W2973062255","https://openalex.org/W2995181338","https://openalex.org/W3015826515","https://openalex.org/W3016185664","https://openalex.org/W3096408984","https://openalex.org/W3097777922","https://openalex.org/W3161480375","https://openalex.org/W3194338569","https://openalex.org/W3196584150","https://openalex.org/W3206706278","https://openalex.org/W4281820413","https://openalex.org/W4307323391","https://openalex.org/W4312933868","https://openalex.org/W4313679638","https://openalex.org/W4318351475","https://openalex.org/W4320458302","https://openalex.org/W4323651091","https://openalex.org/W4367597591","https://openalex.org/W4378945745","https://openalex.org/W4379924545","https://openalex.org/W4381786045","https://openalex.org/W4381827575","https://openalex.org/W4385245566","https://openalex.org/W4386072096","https://openalex.org/W4390075359","https://openalex.org/W4390962167","https://openalex.org/W4392538788","https://openalex.org/W4392908343","https://openalex.org/W4401042948","https://openalex.org/W6688816777","https://openalex.org/W6752888775","https://openalex.org/W6778883912","https://openalex.org/W6790978476","https://openalex.org/W6810334672","https://openalex.org/W6848735303","https://openalex.org/W6849105126","https://openalex.org/W6850204008","https://openalex.org/W6850334629","https://openalex.org/W6851724922","https://openalex.org/W6852870047","https://openalex.org/W6853165267","https://openalex.org/W6853771913","https://openalex.org/W6853888607","https://openalex.org/W6853998256","https://openalex.org/W6861001475","https://openalex.org/W6862144568"],"related_works":["https://openalex.org/W4252424250","https://openalex.org/W2370747337","https://openalex.org/W1890500690","https://openalex.org/W2157819213","https://openalex.org/W2205640666","https://openalex.org/W2133351881","https://openalex.org/W2382399415","https://openalex.org/W2369511196","https://openalex.org/W2163719598","https://openalex.org/W4241950017"],"abstract_inverted_index":{"Recent":[0],"advancements":[1],"in":[2,24,39,94,105],"generative":[3],"speech":[4,28,34,49,58,95,116,119],"models":[5,20,133],"based":[6],"on":[7],"audio-text":[8,27],"prompts":[9],"have":[10],"enabled":[11],"remarkable":[12],"innovations":[13],"like":[14],"high-quality":[15],"zero-shot":[16,54,109],"text-to-speech.":[17],"However,":[18],"existing":[19],"still":[21],"face":[22],"limitations":[23],"handling":[25],"diverse":[26],"generation":[29,50],"tasks":[30],"involving":[31],"transforming":[32],"input":[33,93],"and":[35,56,65,82,85,97,118],"processing":[36],"audio":[37],"captured":[38],"adverse":[40],"acoustic":[41],"conditions.":[42],"This":[43],"paper":[44],"introduces":[45],"SpeechX,":[46],"a":[47,87],"versatile":[48],"model":[51],"capable":[52],"of":[53],"TTS":[55],"various":[57,106],"transformation":[59,98],"tasks,":[60,107],"dealing":[61],"with":[62,74,121],"both":[63],"clean":[64],"noisy":[66],"signals.":[67],"SpeechX":[68],"combines":[69],"neural":[70],"codec":[71],"language":[72],"modeling":[73,84],"multi-task":[75],"learning":[76],"using":[77],"task-dependent":[78],"prompting,":[79],"enabling":[80],"unified":[81],"extensible":[83],"providing":[86],"consistent":[88],"way":[89],"for":[90],"leveraging":[91],"textual":[92],"enhancement":[96],"tasks.":[99,135],"Experimental":[100],"results":[101],"show":[102],"SpeechX's":[103],"efficacy":[104],"including":[108],"TTS,":[110],"noise":[111],"suppression,":[112],"target":[113],"speaker":[114],"extraction,":[115],"removal,":[117],"editing":[120],"or":[122,128],"without":[123],"background":[124],"noise,":[125],"achieving":[126],"comparable":[127],"superior":[129],"performance":[130],"to":[131],"specialized":[132],"across":[134]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":22},{"year":2024,"cited_by_count":8}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}