{"id":"https://openalex.org/W4386132131","doi":"https://doi.org/10.1109/lsp.2023.3308474","title":"LM-VC: Zero-Shot Voice Conversion via Speech Generation Based on Language Models","display_name":"LM-VC: Zero-Shot Voice Conversion via Speech Generation Based on Language Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4386132131","doi":"https://doi.org/10.1109/lsp.2023.3308474"},"language":"en","primary_location":{"id":"doi:10.1109/lsp.2023.3308474","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lsp.2023.3308474","pdf_url":null,"source":{"id":"https://openalex.org/S120629676","display_name":"IEEE Signal Processing Letters","issn_l":"1070-9908","issn":["1070-9908","1558-2361"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Signal Processing Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5106698700","display_name":"Zhichao Wang","orcid":"https://orcid.org/0000-0001-8075-1784"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhichao Wang","raw_affiliation_strings":["ASLP Lab, School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-8075-1784","affiliations":[{"raw_affiliation_string":"ASLP Lab, School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055175414","display_name":"Yuanzhe Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuanzhe Chen","raw_affiliation_strings":["ByteDance SAMI Group, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance SAMI Group, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100668966","display_name":"Lei Xie","orcid":"https://orcid.org/0000-0001-8234-0823"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Xie","raw_affiliation_strings":["ASLP Lab, School of Computer Science, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-8234-0823","affiliations":[{"raw_affiliation_string":"ASLP Lab, School of Computer Science, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018710096","display_name":"Qiao Tian","orcid":"https://orcid.org/0000-0001-8177-7724"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiao Tian","raw_affiliation_strings":["ByteDance SAMI Group, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance SAMI Group, Shanghai, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100339106","display_name":"Yu\u2010Ping Wang","orcid":"https://orcid.org/0000-0001-9340-5864"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuping Wang","raw_affiliation_strings":["ByteDance SAMI Group, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance SAMI Group, Shanghai, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2345,"currency":"USD","value_usd":2345},"apc_paid":null,"fwci":3.1501,"has_fulltext":false,"cited_by_count":24,"citation_normalized_percentile":{"value":0.9317254,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"30","issue":null,"first_page":"1157","last_page":"1161"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10863","display_name":"Voice and Speech Disorders","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/2737","display_name":"Physiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8143818378448486},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.6690618991851807},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.44885537028312683},{"id":"https://openalex.org/keywords/speech-synthesis","display_name":"Speech synthesis","score":0.42603519558906555},{"id":"https://openalex.org/keywords/audio-mining","display_name":"Audio mining","score":0.4190725088119507},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.39393725991249084},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3933229148387909},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.3764346241950989},{"id":"https://openalex.org/keywords/voice-activity-detection","display_name":"Voice activity detection","score":0.3402567505836487}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8143818378448486},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.6690618991851807},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.44885537028312683},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.42603519558906555},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.4190725088119507},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39393725991249084},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3933229148387909},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.3764346241950989},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.3402567505836487}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lsp.2023.3308474","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lsp.2023.3308474","pdf_url":null,"source":{"id":"https://openalex.org/S120629676","display_name":"IEEE Signal Processing Letters","issn_l":"1070-9908","issn":["1070-9908","1558-2361"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Signal Processing Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4099999964237213,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W1494198834","https://openalex.org/W1572989473","https://openalex.org/W1915251500","https://openalex.org/W2091318760","https://openalex.org/W2518172956","https://openalex.org/W2972359262","https://openalex.org/W2972659941","https://openalex.org/W3024869864","https://openalex.org/W3138516171","https://openalex.org/W3152740956","https://openalex.org/W3154308281","https://openalex.org/W3161627112","https://openalex.org/W3162390194","https://openalex.org/W3197358873","https://openalex.org/W3197659778","https://openalex.org/W3198082505","https://openalex.org/W3202267900","https://openalex.org/W3209059054","https://openalex.org/W3215615641","https://openalex.org/W4221141917","https://openalex.org/W4285294723","https://openalex.org/W4296069138","https://openalex.org/W4297808394","https://openalex.org/W4313679638","https://openalex.org/W4318351475","https://openalex.org/W4376632433","https://openalex.org/W4376632463","https://openalex.org/W4381786045","https://openalex.org/W4384648639","https://openalex.org/W4385245566","https://openalex.org/W4390075359","https://openalex.org/W6603838645","https://openalex.org/W6634186343","https://openalex.org/W6640059789","https://openalex.org/W6762122294","https://openalex.org/W6762533536","https://openalex.org/W6776390925","https://openalex.org/W6780218876","https://openalex.org/W6794185780","https://openalex.org/W6802029025","https://openalex.org/W6803547063","https://openalex.org/W6805710207","https://openalex.org/W6810447587","https://openalex.org/W6844194202","https://openalex.org/W6848735303","https://openalex.org/W6849105126","https://openalex.org/W6852503157","https://openalex.org/W6852638271","https://openalex.org/W6854308872","https://openalex.org/W6936113694"],"related_works":["https://openalex.org/W2164147372","https://openalex.org/W4253660971","https://openalex.org/W2550171623","https://openalex.org/W596245619","https://openalex.org/W642007152","https://openalex.org/W2131711534","https://openalex.org/W2184127972","https://openalex.org/W2343205865","https://openalex.org/W2341426843","https://openalex.org/W2009814707"],"abstract_inverted_index":{"Language":[0],"model":[1,212],"(LM)":[2],"based":[3,228],"audio":[4,17],"generation":[5,127],"frameworks,":[6],"e.g.,":[7],"AudioLM,":[8],"have":[9],"recently":[10],"achieved":[11],"new":[12],"state-of-the-art":[13],"performance":[14],"in":[15,80,94,129,245,265,286,297],"zero-shot":[16],"generation.":[18],"In":[19],"this":[20],"paper,":[21],"we":[22,154],"explore":[23],"the":[24,65,76,90,95,105,117,121,126,130,133,168,179,217,221,230,242,246,257,266,282,287],"feasibility":[25],"of":[26,64,132],"LMs":[27],"forzero-shot":[28],"voice":[29,96],"conversion.":[30],"An":[31],"intuitive":[32],"approach":[33,71,160],"is":[34,205,213,261],"to":[35,56,116,137,143,188,215,239,255,263],"follow":[36],"AudioLM":[37],"\u2013":[38],"Tokenizing":[39],"speech":[40,92,147,227,298],"into":[41],"semantic":[42,54,81,106,236],"and":[43,49,51,146,172,176,192,224,234,284,300],"acoustic":[44,58,62,164,182,209,259,268,279],"tokens":[45,55,59,63,82,107,165,280],"respectively":[46],"by":[47],"HuBERT":[48],"SoundStream,":[50],"converting":[52],"source":[53,169],"target":[57,66,118,122,173,226,231],"conditioned":[60],"on":[61,229],"speaker.":[67],"However,":[68],"such":[69],"an":[70,248],"encounters":[72],"several":[73],"issues:":[74],"1)":[75],"linguistic":[77,170],"content":[78,171,190,219],"contained":[79],"may":[83,113],"get":[84],"dispersed":[85],"during":[86,140],"multi-layer":[87],"modeling":[88,159,269],"while":[89],"lengthy":[91],"input":[93],"conversion":[97],"task":[98],"makes":[99],"contextual":[100],"learning":[101],"even":[102],"harder;":[103],"2)":[104],"still":[108],"contain":[109],"speaker-related":[110],"information,":[111],"which":[112,251],"be":[114],"leaked":[115],"speech,":[119],"lowering":[120],"speaker":[123,301],"similarity;":[124],"3)":[125],"diversity":[128],"sampling":[131,243],"LM":[134,199,276],"can":[135],"lead":[136],"unexpected":[138],"outcomes":[139],"inference,":[141],"leading":[142],"unnatural":[144],"pronunciation":[145],"quality":[148],"degradation.":[149],"To":[150],"mitigate":[151],"these":[152],"problems,":[153],"proposeLM-VC,":[155],"a":[156,196,201,274],"two-stage":[157],"language":[158],"that":[161,292],"generates":[162],"coarse":[163,208,267,283],"for":[166,181,207],"recovering":[167],"speaker's":[174,232],"timbre,":[175],"then":[177],"reconstructs":[178,277],"fine":[180,278],"details":[183],"as":[184],"converted":[185,288],"speech.":[186,289],"Specifically,":[187],"enhance":[189],"preservation":[191],"facilitates":[193],"better":[194],"disentanglement,":[195],"masked":[197,218],"prefix":[198,275],"with":[200],"mask":[202],"prediction":[203],"strategy":[204],"used":[206],"modeling.":[210],"This":[211],"encouraged":[214],"recover":[216],"from":[220,281],"surrounding":[222],"context":[223],"generate":[225],"utterance":[233],"corrupted":[235],"tokens.":[237],"Besides,":[238],"further":[240],"alleviate":[241],"error":[244],"generation,":[247],"external":[249],"LM,":[250],"employs":[252],"window":[253],"attention":[254],"capture":[256],"local":[258],"relations,":[260],"introduced":[262],"participate":[264],"through":[270],"shallow":[271],"fusion.":[272],"Finally,":[273],"results":[285],"Experiments":[290],"demonstrate":[291],"LM-VC":[293],"outperforms":[294],"competitive":[295],"systems":[296],"naturalness":[299],"similarity.":[302]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":11}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2025-10-10T00:00:00"}